diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 5baee64e..ebcf8709 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -960,7 +960,6 @@ void AdnlPeerPairImpl::got_data_from_dht(td::Result R) { CHECK(dht_query_active_); dht_query_active_ = false; next_dht_query_at_ = td::Timestamp::in(td::Random::fast(60.0, 120.0)); - alarm_timestamp().relax(next_dht_query_at_); if (R.is_error()) { VLOG(ADNL_INFO) << this << ": dht query failed: " << R.move_as_error(); return; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 47ea4e47..fcabf0c0 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -704,7 +704,9 @@ bool Transaction::prepare_storage_phase(const StoragePhaseConfig& cfg, bool forc switch (acc_status) { case Account::acc_uninit: case Account::acc_frozen: - if (total_due > cfg.delete_due_limit) { + if (total_due > cfg.delete_due_limit && balance.extra.is_null()) { + // Keeping accounts with non-null extras is a temporary measure before implementing proper collection of + // extracurrencies from deleted accounts res->deleted = true; acc_status = Account::acc_deleted; if (balance.extra.not_null()) { diff --git a/crypto/vm/db/CellHashTable.h b/crypto/vm/db/CellHashTable.h index a251c4db..7d0308b7 100644 --- a/crypto/vm/db/CellHashTable.h +++ b/crypto/vm/db/CellHashTable.h @@ -43,7 +43,7 @@ class CellHashTable { template void for_each(F &&f) { for (auto &info : set_) { - f(const_cast(info)); + f(info); } } template diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index 470c46a0..a1b7365b 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -150,20 +150,6 @@ td::Result CellLoader::load(td::Slice hash, bool need_da return res; } -td::Result CellLoader::load_refcnt(td::Slice hash) { - LoadResult res; - std::string serialized; - TRY_RESULT(get_status, reader_->get(hash, serialized)); - if (get_status != KeyValue::GetStatus::Ok) { - DCHECK(get_status == KeyValue::GetStatus::NotFound); - return res; - } - res.status = LoadResult::Ok; - td::TlParser parser(serialized); - td::parse(res.refcnt_, parser); - return res; -} - CellStorer::CellStorer(KeyValue &kv) : kv_(kv) { } diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index ba93ce20..b705b531 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -48,7 +48,6 @@ class CellLoader { }; CellLoader(std::shared_ptr reader); td::Result load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator); - td::Result load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null private: std::shared_ptr reader_; diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index f9fe69cb..0d1d099f 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -27,9 +27,6 @@ #include "td/utils/ThreadSafeCounter.h" #include "vm/cellslice.h" -#include -#include "td/actor/actor.h" -#include "common/delay.h" namespace vm { namespace { @@ -558,177 +555,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat DynamicBocExtCellExtra{cell_db_reader_})); return std::move(res); } - - struct PrepareCommitAsyncState { - size_t remaining_ = 0; - std::shared_ptr executor_; - td::Promise promise_; - - struct CellInfo2 { - CellInfo *info; - std::vector parents; - unsigned remaining_children = 0; - Cell::Hash key() const { - return info->key(); - } - bool operator<(const CellInfo2 &other) const { - return key() < other.key(); - } - - friend bool operator<(const CellInfo2 &a, td::Slice b) { - return a.key().as_slice() < b; - } - - friend bool operator<(td::Slice a, const CellInfo2 &b) { - return a < b.key().as_slice(); - } - }; - - CellHashTable cells_; - }; - std::unique_ptr pca_state_; - - void prepare_commit_async(std::shared_ptr executor, td::Promise promise) override { - if (pca_state_) { - promise.set_error(td::Status::Error("Other prepare_commit_async is not finished")); - return; - } - if (is_prepared_for_commit()) { - promise.set_result(td::Unit()); - return; - } - pca_state_ = std::make_unique(); - pca_state_->executor_ = std::move(executor); - pca_state_->promise_ = std::move(promise); - for (auto &new_cell : to_inc_) { - dfs_new_cells_in_db_async(new_cell); - } - pca_state_->cells_.for_each([&](PrepareCommitAsyncState::CellInfo2 &info) { - ++pca_state_->remaining_; - if (info.remaining_children == 0) { - pca_load_from_db(&info); - } - }); - if (pca_state_->remaining_ == 0) { - prepare_commit_async_cont(); - } - } - - void dfs_new_cells_in_db_async(const td::Ref &cell, PrepareCommitAsyncState::CellInfo2 *parent = nullptr) { - bool exists = true; - pca_state_->cells_.apply(cell->get_hash().as_slice(), [&](PrepareCommitAsyncState::CellInfo2 &info) { - if (info.info == nullptr) { - exists = false; - info.info = &get_cell_info(cell); - } - }); - auto info = pca_state_->cells_.get_if_exists(cell->get_hash().as_slice()); - if (parent) { - info->parents.push_back(parent); - ++parent->remaining_children; - } - if (exists) { - return; - } - if (cell->is_loaded()) { - vm::CellSlice cs(vm::NoVm{}, cell); - for (unsigned i = 0; i < cs.size_refs(); i++) { - dfs_new_cells_in_db_async(cs.prefetch_ref(i), info); - } - } - } - - void pca_load_from_db(PrepareCommitAsyncState::CellInfo2 *info) { - pca_state_->executor_->execute_async( - [db = this, info, executor = pca_state_->executor_, loader = *loader_]() mutable { - auto res = loader.load_refcnt(info->info->cell->get_hash().as_slice()).move_as_ok(); - executor->execute_sync([db, info, res = std::move(res)]() { - db->pca_set_in_db(info, std::move(res)); - }); - }); - } - - void pca_set_in_db(PrepareCommitAsyncState::CellInfo2 *info, CellLoader::LoadResult result) { - info->info->sync_with_db = true; - if (result.status == CellLoader::LoadResult::Ok) { - info->info->in_db = true; - info->info->db_refcnt = result.refcnt(); - } else { - info->info->in_db = false; - } - for (PrepareCommitAsyncState::CellInfo2 *parent_info : info->parents) { - if (parent_info->info->sync_with_db) { - continue; - } - if (!info->info->in_db) { - pca_set_in_db(parent_info, {}); - } else if (--parent_info->remaining_children == 0) { - pca_load_from_db(parent_info); - } - } - if (--pca_state_->remaining_ == 0) { - prepare_commit_async_cont(); - } - } - - void prepare_commit_async_cont() { - for (auto &new_cell : to_inc_) { - auto &new_cell_info = get_cell_info(new_cell); - dfs_new_cells(new_cell_info); - } - - CHECK(pca_state_->remaining_ == 0); - for (auto &old_cell : to_dec_) { - auto &old_cell_info = get_cell_info(old_cell); - dfs_old_cells_async(old_cell_info); - } - if (pca_state_->remaining_ == 0) { - prepare_commit_async_cont2(); - } - } - - void dfs_old_cells_async(CellInfo &info) { - if (!info.was) { - info.was = true; - visited_.push_back(&info); - if (!info.sync_with_db) { - ++pca_state_->remaining_; - load_cell_async( - info.cell->get_hash().as_slice(), pca_state_->executor_, - [executor = pca_state_->executor_, db = this, info = &info](td::Result> R) { - R.ensure(); - executor->execute_sync([db, info]() { - CHECK(info->sync_with_db); - db->dfs_old_cells_async(*info); - if (--db->pca_state_->remaining_ == 0) { - db->prepare_commit_async_cont2(); - } - }); - }); - return; - } - } - info.refcnt_diff--; - if (!info.sync_with_db) { - return; - } - auto new_refcnt = info.refcnt_diff + info.db_refcnt; - CHECK(new_refcnt >= 0); - if (new_refcnt != 0) { - return; - } - - for_each(info, [this](auto &child_info) { dfs_old_cells_async(child_info); }); - } - - void prepare_commit_async_cont2() { - save_diff_prepare(); - to_inc_.clear(); - to_dec_.clear(); - pca_state_->promise_.set_result(td::Unit()); - pca_state_ = {}; - } - }; } // namespace diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index 69cab52f..3569208c 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -75,7 +75,6 @@ class DynamicBagOfCellsDb { virtual void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) = 0; - virtual void prepare_commit_async(std::shared_ptr executor, td::Promise promise) = 0; }; } // namespace vm diff --git a/dht/dht-remote-node.cpp b/dht/dht-remote-node.cpp index 653de256..1273750c 100644 --- a/dht/dht-remote-node.cpp +++ b/dht/dht-remote-node.cpp @@ -140,13 +140,7 @@ adnl::AdnlNodeIdFull DhtRemoteNode::get_full_id() const { td::Result> DhtRemoteNode::create(DhtNode node, td::uint32 max_missed_pings, td::int32 our_network_id) { - TRY_RESULT(enc, node.adnl_id().pubkey().create_encryptor()); - auto tl = node.tl(); - auto sig = std::move(tl->signature_); - - TRY_STATUS_PREFIX(enc->check_signature(serialize_tl_object(tl, true).as_slice(), sig.as_slice()), - "bad node signature: "); - + TRY_STATUS(node.check_signature()); return std::make_unique(std::move(node), max_missed_pings, our_network_id); } diff --git a/utils/generate-random-id.cpp b/utils/generate-random-id.cpp index 3727f291..f606f358 100644 --- a/utils/generate-random-id.cpp +++ b/utils/generate-random-id.cpp @@ -28,9 +28,6 @@ #include #include #include -#include -#include -#include "crypto/ellcurve/Ed25519.h" #include "adnl/utils.hpp" #include "auto/tl/ton_api.h" #include "auto/tl/ton_api_json.h" @@ -38,12 +35,13 @@ #include "td/utils/OptionParser.h" #include "td/utils/filesystem.h" #include "keys/encryptor.h" -#include "keys/keys.hpp" #include "git.h" +#include "dht/dht-node.hpp" int main(int argc, char *argv[]) { ton::PrivateKey pk; - ton::tl_object_ptr addr_list; + td::optional addr_list; + td::optional network_id_opt; td::OptionParser p; p.set_description("generate random id"); @@ -78,11 +76,19 @@ int main(int argc, char *argv[]) { if (addr_list) { return td::Status::Error("duplicate '-a' option"); } - CHECK(!addr_list); td::BufferSlice bs(key); TRY_RESULT_PREFIX(as_json_value, td::json_decode(bs.as_slice()), "bad addr list JSON: "); - TRY_STATUS_PREFIX(td::from_json(addr_list, std::move(as_json_value)), "bad addr list TL: "); + ton::tl_object_ptr addr_list_tl; + TRY_STATUS_PREFIX(td::from_json(addr_list_tl, std::move(as_json_value)), "bad addr list TL: "); + TRY_RESULT_PREFIX_ASSIGN(addr_list, ton::adnl::AdnlAddressList::create(addr_list_tl), "bad addr list: "); + return td::Status::OK(); + }); + p.add_checked_option('i', "network-id", "dht network id (default: -1)", [&](td::Slice key) { + if (network_id_opt) { + return td::Status::Error("duplicate '-i' option"); + } + TRY_RESULT_PREFIX_ASSIGN(network_id_opt, td::to_integer_safe(key), "bad network id: "); return td::Status::OK(); }); @@ -118,7 +124,7 @@ int main(int argc, char *argv[]) { std::cerr << "'-a' option missing" << std::endl; return 2; } - auto x = ton::create_tl_object(pub_key.tl(), std::move(addr_list)); + auto x = ton::create_tl_object(pub_key.tl(), addr_list.value().tl()); auto e = pk.create_decryptor().move_as_ok(); auto r = e->sign(ton::serialize_tl_object(x, true).as_slice()).move_as_ok(); @@ -129,12 +135,17 @@ int main(int argc, char *argv[]) { std::cerr << "'-a' option missing" << std::endl; return 2; } - auto x = ton::create_tl_object(pub_key.tl(), std::move(addr_list), -1, td::BufferSlice()); + td::int32 network_id = network_id_opt ? network_id_opt.value() : -1; + td::BufferSlice to_sign = ton::serialize_tl_object( + ton::dht::DhtNode{ton::adnl::AdnlNodeIdFull{pub_key}, addr_list.value(), -1, network_id, td::BufferSlice{}} + .tl(), + true); auto e = pk.create_decryptor().move_as_ok(); - auto r = e->sign(ton::serialize_tl_object(x, true).as_slice()).move_as_ok(); - x->signature_ = std::move(r); + auto signature = e->sign(to_sign.as_slice()).move_as_ok(); + auto node = + ton::dht::DhtNode{ton::adnl::AdnlNodeIdFull{pub_key}, addr_list.value(), -1, network_id, std::move(signature)}; - auto v = td::json_encode(td::ToJson(x)); + auto v = td::json_encode(td::ToJson(node.tl())); std::cout << v << "\n"; } else if (mode == "keys") { td::write_file(name, pk.export_as_slice()).ensure(); diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index c633e395..3b2a34f3 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -86,42 +86,19 @@ void CellDbIn::start_up() { } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { - enqueue([this, hash, promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - return; - } - promise.set_result(boc_->load_cell(hash.as_slice())); - release_db(); - }); + boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise)); } void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise) { - enqueue([this, block_id, cell = std::move(cell), promise = std::move(promise)](td::Result R0) mutable { - if (R0.is_error()) { - return; - } - promise = promise.wrap([timer = td::PerfWarningTimer{"storecell", 0.1}](td::Ref &&r) { return r; }); - auto key_hash = get_key_hash(block_id); - auto R = get_block(key_hash); - // duplicate - if (R.is_ok()) { - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); - release_db(); - return; - } - - boc_->inc(cell); - boc_->prepare_commit_async( - async_executor, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - R.ensure(); - td::actor::send_closure(SelfId, &CellDbIn::store_cell_cont, block_id, cell, std::move(promise)); - }); - }); -} - -void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref cell, - td::Promise> promise) { + td::PerfWarningTimer timer{"storecell", 0.1}; auto key_hash = get_key_hash(block_id); + auto R = get_block(key_hash); + // duplicate + if (R.is_ok()) { + promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + return; + } + auto empty = get_empty_key_hash(); auto ER = get_block(empty); ER.ensure(); @@ -142,7 +119,9 @@ void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref cell, P.prev = key_hash; } - vm::CellStorer stor{*cell_db_}; + boc_->inc(cell); + boc_->prepare_commit().ensure(); + vm::CellStorer stor{*cell_db_.get()}; cell_db_->begin_write_batch().ensure(); boc_->commit(stor).ensure(); set_block(empty, std::move(E)); @@ -154,17 +133,10 @@ void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref cell, td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); - release_db(); } void CellDbIn::get_cell_db_reader(td::Promise> promise) { - enqueue([this, promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - return; - } - promise.set_result(boc_->get_cell_db_reader()); - release_db(); - }); + promise.set_result(boc_->get_cell_db_reader()); } void CellDbIn::alarm() { @@ -214,30 +186,13 @@ void CellDbIn::gc_cont(BlockHandle handle) { } void CellDbIn::gc_cont2(BlockHandle handle) { - enqueue([this, handle = std::move(handle)](td::Result R) mutable { - if (R.is_error()) { - return; - } - td::Promise promise = [timer = td::PerfWarningTimer{"gccell", 0.1}](td::Result) {}; - auto FR = get_block(get_key_hash(handle->id())); - FR.ensure(); - auto F = FR.move_as_ok(); - auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok(); + td::PerfWarningTimer timer{"gccell", 0.1}; - boc_->dec(cell); - boc_->prepare_commit_async(async_executor, [SelfId = actor_id(this), promise = std::move(promise), - block_id = handle->id()](td::Result R) mutable { - R.ensure(); - td::actor::send_closure(SelfId, &CellDbIn::gc_cont3, block_id, std::move(promise)); - }); - }); -} - -void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise promise) { - auto key_hash = get_key_hash(block_id); + auto key_hash = get_key_hash(handle->id()); auto FR = get_block(key_hash); FR.ensure(); auto F = FR.move_as_ok(); + auto PR = get_block(F.prev); PR.ensure(); auto P = PR.move_as_ok(); @@ -252,6 +207,10 @@ void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise promise) { N.next = N.prev; } + auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok(); + + boc_->dec(cell); + boc_->prepare_commit().ensure(); vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); boc_->commit(stor).ensure(); @@ -265,28 +224,6 @@ void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise promise) { td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); DCHECK(get_block(key_hash).is_error()); - promise.set_result(td::Unit()); - release_db(); -} - -void CellDbIn::enqueue(td::Promise promise) { - db_queue_.push(std::move(promise)); - process_event(); -} - -void CellDbIn::release_db() { - db_busy_ = false; - process_event(); -} - -void CellDbIn::process_event() { - if (db_busy_ || db_queue_.empty()) { - return; - } - db_busy_ = true; - auto promise = std::move(db_queue_.front()); - db_queue_.pop(); - promise.set_result(td::Unit()); } void CellDbIn::skip_gc() { diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 08d5a184..a05e9ddb 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -25,7 +25,6 @@ #include "ton/ton-types.h" #include "interfaces/block-handle.h" #include "auto/tl/ton_api.h" -#include namespace ton { @@ -88,11 +87,8 @@ class CellDbIn : public CellDbBase { void gc(BlockIdExt block_id); void gc_cont(BlockHandle handle); void gc_cont2(BlockHandle handle); - void gc_cont3(BlockIdExt block_id, td::Promise promise); void skip_gc(); - void store_cell_cont(BlockIdExt block_id, td::Ref cell, td::Promise> promise); - td::actor::ActorId root_db_; td::actor::ActorId parent_; @@ -100,13 +96,6 @@ class CellDbIn : public CellDbBase { std::unique_ptr boc_; std::shared_ptr cell_db_; - - std::queue> db_queue_; - bool db_busy_ = false; - - void enqueue(td::Promise promise); - void release_db(); - void process_event(); }; class CellDb : public CellDbBase {