external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Merge branch 'testnet' into block-generation

Browse source
This commit is contained in:
SpyCheese 2023-03-31 13:11:36 +03:00
commit 876c68c5c7
11 changed files with 48 additions and 306 deletions
Download patch file Download diff file Expand all files Collapse all files

1
adnl/adnl-peer.cpp
View file

@ -960,7 +960,6 @@ void AdnlPeerPairImpl::got_data_from_dht(td::Result<AdnlNode> R) {
CHECK(dht_query_active_);
dht_query_active_ = false;
next_dht_query_at_ = td::Timestamp::in(td::Random::fast(60.0, 120.0));
alarm_timestamp().relax(next_dht_query_at_);
if (R.is_error()) {
VLOG(ADNL_INFO) << this << ": dht query failed: " << R.move_as_error();
return;

4
crypto/block/transaction.cpp
View file

@ -704,7 +704,9 @@ bool Transaction::prepare_storage_phase(const StoragePhaseConfig& cfg, bool forc
switch (acc_status) {
case Account::acc_uninit:
case Account::acc_frozen:
if (total_due > cfg.delete_due_limit) {
if (total_due > cfg.delete_due_limit && balance.extra.is_null()) {
// Keeping accounts with non-null extras is a temporary measure before implementing proper collection of
// extracurrencies from deleted accounts
res->deleted = true;
acc_status = Account::acc_deleted;
if (balance.extra.not_null()) {

2
crypto/vm/db/CellHashTable.h
View file

@ -43,7 +43,7 @@ class CellHashTable {
template <class F>
void for_each(F &&f) {
for (auto &info : set_) {
f(const_cast<InfoT &>(info));
f(info);
}
}
template <class F>

14
crypto/vm/db/CellStorage.cpp
View file

@ -150,20 +150,6 @@ td::Result<CellLoader::LoadResult> CellLoader::load(td::Slice hash, bool need_da
return res;
}
td::Result<CellLoader::LoadResult> CellLoader::load_refcnt(td::Slice hash) {
LoadResult res;
std::string serialized;
TRY_RESULT(get_status, reader_->get(hash, serialized));
if (get_status != KeyValue::GetStatus::Ok) {
DCHECK(get_status == KeyValue::GetStatus::NotFound);
return res;
}
res.status = LoadResult::Ok;
td::TlParser parser(serialized);
td::parse(res.refcnt_, parser);
return res;
}
CellStorer::CellStorer(KeyValue &kv) : kv_(kv) {
}

1
crypto/vm/db/CellStorage.h
View file

@ -48,7 +48,6 @@ class CellLoader {
};
CellLoader(std::shared_ptr<KeyValueReader> reader);
td::Result<LoadResult> load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator);
td::Result<LoadResult> load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null
private:
std::shared_ptr<KeyValueReader> reader_;

174
crypto/vm/db/DynamicBagOfCellsDb.cpp
View file

@ -27,9 +27,6 @@
#include "td/utils/ThreadSafeCounter.h"
#include "vm/cellslice.h"
#include <queue>
#include "td/actor/actor.h"
#include "common/delay.h"
namespace vm {
namespace {
@ -558,177 +555,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
DynamicBocExtCellExtra{cell_db_reader_}));
return std::move(res);
}
struct PrepareCommitAsyncState {
size_t remaining_ = 0;
std::shared_ptr<AsyncExecutor> executor_;
td::Promise<td::Unit> promise_;
struct CellInfo2 {
CellInfo *info;
std::vector<CellInfo2 *> parents;
unsigned remaining_children = 0;
Cell::Hash key() const {
return info->key();
}
bool operator<(const CellInfo2 &other) const {
return key() < other.key();
}
friend bool operator<(const CellInfo2 &a, td::Slice b) {
return a.key().as_slice() < b;
}
friend bool operator<(td::Slice a, const CellInfo2 &b) {
return a < b.key().as_slice();
}
};
CellHashTable<CellInfo2> cells_;
};
std::unique_ptr<PrepareCommitAsyncState> pca_state_;
void prepare_commit_async(std::shared_ptr<AsyncExecutor> executor, td::Promise<td::Unit> promise) override {
if (pca_state_) {
promise.set_error(td::Status::Error("Other prepare_commit_async is not finished"));
return;
}
if (is_prepared_for_commit()) {
promise.set_result(td::Unit());
return;
}
pca_state_ = std::make_unique<PrepareCommitAsyncState>();
pca_state_->executor_ = std::move(executor);
pca_state_->promise_ = std::move(promise);
for (auto &new_cell : to_inc_) {
dfs_new_cells_in_db_async(new_cell);
}
pca_state_->cells_.for_each([&](PrepareCommitAsyncState::CellInfo2 &info) {
++pca_state_->remaining_;
if (info.remaining_children == 0) {
pca_load_from_db(&info);
}
});
if (pca_state_->remaining_ == 0) {
prepare_commit_async_cont();
}
}
void dfs_new_cells_in_db_async(const td::Ref<vm::Cell> &cell, PrepareCommitAsyncState::CellInfo2 *parent = nullptr) {
bool exists = true;
pca_state_->cells_.apply(cell->get_hash().as_slice(), [&](PrepareCommitAsyncState::CellInfo2 &info) {
if (info.info == nullptr) {
exists = false;
info.info = &get_cell_info(cell);
}
});
auto info = pca_state_->cells_.get_if_exists(cell->get_hash().as_slice());
if (parent) {
info->parents.push_back(parent);
++parent->remaining_children;
}
if (exists) {
return;
}
if (cell->is_loaded()) {
vm::CellSlice cs(vm::NoVm{}, cell);
for (unsigned i = 0; i < cs.size_refs(); i++) {
dfs_new_cells_in_db_async(cs.prefetch_ref(i), info);
}
}
}
void pca_load_from_db(PrepareCommitAsyncState::CellInfo2 *info) {
pca_state_->executor_->execute_async(
[db = this, info, executor = pca_state_->executor_, loader = *loader_]() mutable {
auto res = loader.load_refcnt(info->info->cell->get_hash().as_slice()).move_as_ok();
executor->execute_sync([db, info, res = std::move(res)]() {
db->pca_set_in_db(info, std::move(res));
});
});
}
void pca_set_in_db(PrepareCommitAsyncState::CellInfo2 *info, CellLoader::LoadResult result) {
info->info->sync_with_db = true;
if (result.status == CellLoader::LoadResult::Ok) {
info->info->in_db = true;
info->info->db_refcnt = result.refcnt();
} else {
info->info->in_db = false;
}
for (PrepareCommitAsyncState::CellInfo2 *parent_info : info->parents) {
if (parent_info->info->sync_with_db) {
continue;
}
if (!info->info->in_db) {
pca_set_in_db(parent_info, {});
} else if (--parent_info->remaining_children == 0) {
pca_load_from_db(parent_info);
}
}
if (--pca_state_->remaining_ == 0) {
prepare_commit_async_cont();
}
}
void prepare_commit_async_cont() {
for (auto &new_cell : to_inc_) {
auto &new_cell_info = get_cell_info(new_cell);
dfs_new_cells(new_cell_info);
}
CHECK(pca_state_->remaining_ == 0);
for (auto &old_cell : to_dec_) {
auto &old_cell_info = get_cell_info(old_cell);
dfs_old_cells_async(old_cell_info);
}
if (pca_state_->remaining_ == 0) {
prepare_commit_async_cont2();
}
}
void dfs_old_cells_async(CellInfo &info) {
if (!info.was) {
info.was = true;
visited_.push_back(&info);
if (!info.sync_with_db) {
++pca_state_->remaining_;
load_cell_async(
info.cell->get_hash().as_slice(), pca_state_->executor_,
[executor = pca_state_->executor_, db = this, info = &info](td::Result<td::Ref<vm::DataCell>> R) {
R.ensure();
executor->execute_sync([db, info]() {
CHECK(info->sync_with_db);
db->dfs_old_cells_async(*info);
if (--db->pca_state_->remaining_ == 0) {
db->prepare_commit_async_cont2();
}
});
});
return;
}
}
info.refcnt_diff--;
if (!info.sync_with_db) {
return;
}
auto new_refcnt = info.refcnt_diff + info.db_refcnt;
CHECK(new_refcnt >= 0);
if (new_refcnt != 0) {
return;
}
for_each(info, [this](auto &child_info) { dfs_old_cells_async(child_info); });
}
void prepare_commit_async_cont2() {
save_diff_prepare();
to_inc_.clear();
to_dec_.clear();
pca_state_->promise_.set_result(td::Unit());
pca_state_ = {};
}
};
} // namespace

1
crypto/vm/db/DynamicBagOfCellsDb.h
View file

@ -75,7 +75,6 @@ class DynamicBagOfCellsDb {
virtual void load_cell_async(td::Slice hash, std::shared_ptr<AsyncExecutor> executor,
td::Promise<Ref<DataCell>> promise) = 0;
virtual void prepare_commit_async(std::shared_ptr<AsyncExecutor> executor, td::Promise<td::Unit> promise) = 0;
};
} // namespace vm

8
dht/dht-remote-node.cpp
View file

@ -140,13 +140,7 @@ adnl::AdnlNodeIdFull DhtRemoteNode::get_full_id() const {
td::Result<std::unique_ptr<DhtRemoteNode>> DhtRemoteNode::create(DhtNode node, td::uint32 max_missed_pings,
td::int32 our_network_id) {
TRY_RESULT(enc, node.adnl_id().pubkey().create_encryptor());
auto tl = node.tl();
auto sig = std::move(tl->signature_);
TRY_STATUS_PREFIX(enc->check_signature(serialize_tl_object(tl, true).as_slice(), sig.as_slice()),
"bad node signature: ");
TRY_STATUS(node.check_signature());
return std::make_unique<DhtRemoteNode>(std::move(node), max_missed_pings, our_network_id);
}

35
utils/generate-random-id.cpp
View file

@ -28,9 +28,6 @@
#include <iostream>
#include <iomanip>
#include <string>
#include <cstring>
#include <cassert>
#include "crypto/ellcurve/Ed25519.h"
#include "adnl/utils.hpp"
#include "auto/tl/ton_api.h"
#include "auto/tl/ton_api_json.h"
@ -38,12 +35,13 @@
#include "td/utils/OptionParser.h"
#include "td/utils/filesystem.h"
#include "keys/encryptor.h"
#include "keys/keys.hpp"
#include "git.h"
#include "dht/dht-node.hpp"
int main(int argc, char *argv[]) {
ton::PrivateKey pk;
ton::tl_object_ptr<ton::ton_api::adnl_addressList> addr_list;
td::optional<ton::adnl::AdnlAddressList> addr_list;
td::optional<td::int32> network_id_opt;
td::OptionParser p;
p.set_description("generate random id");
@ -78,11 +76,19 @@ int main(int argc, char *argv[]) {
if (addr_list) {
return td::Status::Error("duplicate '-a' option");
}
CHECK(!addr_list);
td::BufferSlice bs(key);
TRY_RESULT_PREFIX(as_json_value, td::json_decode(bs.as_slice()), "bad addr list JSON: ");
TRY_STATUS_PREFIX(td::from_json(addr_list, std::move(as_json_value)), "bad addr list TL: ");
ton::tl_object_ptr<ton::ton_api::adnl_addressList> addr_list_tl;
TRY_STATUS_PREFIX(td::from_json(addr_list_tl, std::move(as_json_value)), "bad addr list TL: ");
TRY_RESULT_PREFIX_ASSIGN(addr_list, ton::adnl::AdnlAddressList::create(addr_list_tl), "bad addr list: ");
return td::Status::OK();
});
p.add_checked_option('i', "network-id", "dht network id (default: -1)", [&](td::Slice key) {
if (network_id_opt) {
return td::Status::Error("duplicate '-i' option");
}
TRY_RESULT_PREFIX_ASSIGN(network_id_opt, td::to_integer_safe<td::int32>(key), "bad network id: ");
return td::Status::OK();
});
@ -118,7 +124,7 @@ int main(int argc, char *argv[]) {
std::cerr << "'-a' option missing" << std::endl;
return 2;
}
auto x = ton::create_tl_object<ton::ton_api::adnl_node>(pub_key.tl(), std::move(addr_list));
auto x = ton::create_tl_object<ton::ton_api::adnl_node>(pub_key.tl(), addr_list.value().tl());
auto e = pk.create_decryptor().move_as_ok();
auto r = e->sign(ton::serialize_tl_object(x, true).as_slice()).move_as_ok();
@ -129,12 +135,17 @@ int main(int argc, char *argv[]) {
std::cerr << "'-a' option missing" << std::endl;
return 2;
}
auto x = ton::create_tl_object<ton::ton_api::dht_node>(pub_key.tl(), std::move(addr_list), -1, td::BufferSlice());
td::int32 network_id = network_id_opt ? network_id_opt.value() : -1;
td::BufferSlice to_sign = ton::serialize_tl_object(
ton::dht::DhtNode{ton::adnl::AdnlNodeIdFull{pub_key}, addr_list.value(), -1, network_id, td::BufferSlice{}}
.tl(),
true);
auto e = pk.create_decryptor().move_as_ok();
auto r = e->sign(ton::serialize_tl_object(x, true).as_slice()).move_as_ok();
x->signature_ = std::move(r);
auto signature = e->sign(to_sign.as_slice()).move_as_ok();
auto node =
ton::dht::DhtNode{ton::adnl::AdnlNodeIdFull{pub_key}, addr_list.value(), -1, network_id, std::move(signature)};
auto v = td::json_encode<std::string>(td::ToJson(x));
auto v = td::json_encode<std::string>(td::ToJson(node.tl()));
std::cout << v << "\n";
} else if (mode == "keys") {
td::write_file(name, pk.export_as_slice()).ensure();

103
validator/db/celldb.cpp
View file

@ -86,42 +86,19 @@ void CellDbIn::start_up() {
}
void CellDbIn::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
enqueue([this, hash, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
return;
}
promise.set_result(boc_->load_cell(hash.as_slice()));
release_db();
});
boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise));
}
void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise) {
enqueue([this, block_id, cell = std::move(cell), promise = std::move(promise)](td::Result<td::Unit> R0) mutable {
if (R0.is_error()) {
return;
}
promise = promise.wrap([timer = td::PerfWarningTimer{"storecell", 0.1}](td::Ref<vm::DataCell> &&r) { return r; });
auto key_hash = get_key_hash(block_id);
auto R = get_block(key_hash);
// duplicate
if (R.is_ok()) {
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
release_db();
return;
}
boc_->inc(cell);
boc_->prepare_commit_async(
async_executor, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::Unit> R) mutable {
R.ensure();
td::actor::send_closure(SelfId, &CellDbIn::store_cell_cont, block_id, cell, std::move(promise));
});
});
}
void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell,
td::Promise<td::Ref<vm::DataCell>> promise) {
td::PerfWarningTimer timer{"storecell", 0.1};
auto key_hash = get_key_hash(block_id);
auto R = get_block(key_hash);
// duplicate
if (R.is_ok()) {
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
return;
}
auto empty = get_empty_key_hash();
auto ER = get_block(empty);
ER.ensure();
@ -142,7 +119,9 @@ void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell,
P.prev = key_hash;
}
vm::CellStorer stor{*cell_db_};
boc_->inc(cell);
boc_->prepare_commit().ensure();
vm::CellStorer stor{*cell_db_.get()};
cell_db_->begin_write_batch().ensure();
boc_->commit(stor).ensure();
set_block(empty, std::move(E));
@ -154,17 +133,10 @@ void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell,
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
release_db();
}
void CellDbIn::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) {
enqueue([this, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
return;
}
promise.set_result(boc_->get_cell_db_reader());
release_db();
});
promise.set_result(boc_->get_cell_db_reader());
}
void CellDbIn::alarm() {
@ -214,30 +186,13 @@ void CellDbIn::gc_cont(BlockHandle handle) {
}
void CellDbIn::gc_cont2(BlockHandle handle) {
enqueue([this, handle = std::move(handle)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
return;
}
td::Promise<td::Unit> promise = [timer = td::PerfWarningTimer{"gccell", 0.1}](td::Result<td::Unit>) {};
auto FR = get_block(get_key_hash(handle->id()));
FR.ensure();
auto F = FR.move_as_ok();
auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok();
td::PerfWarningTimer timer{"gccell", 0.1};
boc_->dec(cell);
boc_->prepare_commit_async(async_executor, [SelfId = actor_id(this), promise = std::move(promise),
block_id = handle->id()](td::Result<td::Unit> R) mutable {
R.ensure();
td::actor::send_closure(SelfId, &CellDbIn::gc_cont3, block_id, std::move(promise));
});
});
}
void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise) {
auto key_hash = get_key_hash(block_id);
auto key_hash = get_key_hash(handle->id());
auto FR = get_block(key_hash);
FR.ensure();
auto F = FR.move_as_ok();
auto PR = get_block(F.prev);
PR.ensure();
auto P = PR.move_as_ok();
@ -252,6 +207,10 @@ void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise) {
N.next = N.prev;
}
auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok();
boc_->dec(cell);
boc_->prepare_commit().ensure();
vm::CellStorer stor{*cell_db_};
cell_db_->begin_write_batch().ensure();
boc_->commit(stor).ensure();
@ -265,28 +224,6 @@ void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise) {
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
DCHECK(get_block(key_hash).is_error());
promise.set_result(td::Unit());
release_db();
}
void CellDbIn::enqueue(td::Promise<td::Unit> promise) {
db_queue_.push(std::move(promise));
process_event();
}
void CellDbIn::release_db() {
db_busy_ = false;
process_event();
}
void CellDbIn::process_event() {
if (db_busy_ || db_queue_.empty()) {
return;
}
db_busy_ = true;
auto promise = std::move(db_queue_.front());
db_queue_.pop();
promise.set_result(td::Unit());
}
void CellDbIn::skip_gc() {

11
validator/db/celldb.hpp
View file

@ -25,7 +25,6 @@
#include "ton/ton-types.h"
#include "interfaces/block-handle.h"
#include "auto/tl/ton_api.h"
#include <queue>
namespace ton {
@ -88,11 +87,8 @@ class CellDbIn : public CellDbBase {
void gc(BlockIdExt block_id);
void gc_cont(BlockHandle handle);
void gc_cont2(BlockHandle handle);
void gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise);
void skip_gc();
void store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise);
td::actor::ActorId<RootDb> root_db_;
td::actor::ActorId<CellDb> parent_;
@ -100,13 +96,6 @@ class CellDbIn : public CellDbBase {
std::unique_ptr<vm::DynamicBagOfCellsDb> boc_;
std::shared_ptr<vm::KeyValue> cell_db_;
std::queue<td::Promise<td::Unit>> db_queue_;
bool db_busy_ = false;
void enqueue(td::Promise<td::Unit> promise);
void release_db();
void process_event();
};
class CellDb : public CellDbBase {