From 61ef357941668b51ba7319627c91b180e317d387 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 6 Mar 2025 17:15:27 +0300 Subject: [PATCH] Request persistent state size, add missing queries to full-node-master --- tl/generate/scheme/ton_api.tl | 2 ++ tl/generate/scheme/ton_api.tlo | Bin 102484 -> 102788 bytes validator/db/archive-manager.cpp | 50 +++++++++++++++-------------- validator/db/archive-manager.hpp | 9 ++++-- validator/db/rootdb.cpp | 6 ++-- validator/db/rootdb.hpp | 4 +-- validator/full-node-master.cpp | 37 +++++++++++++++++++--- validator/full-node-master.hpp | 4 +++ validator/full-node-shard.cpp | 22 ++++++++++--- validator/full-node-shard.hpp | 2 ++ validator/interfaces/db.h | 4 +-- validator/manager-disk.cpp | 6 ++-- validator/manager-disk.hpp | 4 +-- validator/manager-hardfork.hpp | 4 +-- validator/manager.cpp | 6 ++-- validator/manager.hpp | 4 +-- validator/net/download-state.cpp | 52 ++++++++++++++++++++++++++++--- validator/net/download-state.hpp | 3 ++ validator/validator.h | 4 +-- 19 files changed, 163 insertions(+), 60 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index cfc9f3a1..14e0121f 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -406,6 +406,7 @@ tonNode.preparedProof = tonNode.PreparedProof; tonNode.preparedProofLink = tonNode.PreparedProof; tonNode.preparedState = tonNode.PreparedState; tonNode.notFoundState = tonNode.PreparedState; +tonNode.persistentStateSize size:long = tonNode.PersistentStateSize; tonNode.prepared = tonNode.Prepared; tonNode.notFound = tonNode.Prepared; tonNode.data data:bytes = tonNode.Data; @@ -472,6 +473,7 @@ tonNode.prepareKeyBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:B tonNode.prepareBlock block:tonNode.blockIdExt = tonNode.Prepared; tonNode.prepareBlocks blocks:(vector tonNode.blockIdExt) = tonNode.Prepared; tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; +tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize; tonNode.prepareZeroState block:tonNode.blockIdExt = tonNode.PreparedState; tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBlocks; tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 96ecb7751b1ad3a31830cba56037d0037988cde3..5d9ccc2d8586fcb9da5e367372b01e3394ff8b62 100644 GIT binary patch delta 191 zcmcbzfURXQ8}Fmp`c@23puUlJwz1-+%POCwOY-yl@>5dv0#b{LGmA@7^GbqC5=&Bp zGpkY=Cm&=I-@L+Dgq?Bo=Dk*n@2jBbD8SSKlAr9b(|`rU*d8Lqn4rhlwSB7x<9r7V g-9PDWaxmM{Q%kT|$NK&7T6}-|A1THVJ;u)Mfu4->9hSFr HF|q>y03{O* diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 8c7cde17..9b1ebf6c 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -314,7 +314,15 @@ void ArchiveManager::register_perm_state(FileReferenceShort id) { BlockSeqno masterchain_seqno = 0; id.ref().visit(td::overloaded( [&](const fileref::PersistentStateShort &x) { masterchain_seqno = x.masterchain_seqno; }, [&](const auto &) {})); - perm_states_[{masterchain_seqno, id.hash()}] = id; + td::uint64 size; + auto r_stat = td::stat(db_root_ + "/archive/states/" + id.filename_short()); + if (r_stat.is_error()) { + LOG(WARNING) << "Cannot stat persistent state file " << id.filename_short() << " : " << r_stat.move_as_error(); + size = 0; + } else { + size = r_stat.ok().size_; + } + perm_states_[{masterchain_seqno, id.hash()}] = {.id = id, .size = size}; } void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise) { @@ -417,7 +425,7 @@ void ArchiveManager::get_previous_persistent_state_files( BlockSeqno mc_seqno = it->first.first; std::vector> files; while (it->first.first == mc_seqno) { - files.emplace_back(db_root_ + "/archive/states/" + it->second.filename_short(), it->second.shard()); + files.emplace_back(db_root_ + "/archive/states/" + it->second.id.filename_short(), it->second.id.shard()); if (it == perm_states_.begin()) { break; } @@ -452,15 +460,16 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt td::actor::create_actor("readfile", path, offset, max_size, 0, std::move(promise)).release(); } -void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { +void ArchiveManager::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); - if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { - promise.set_result(false); + auto it = perm_states_.find({masterchain_block_id.seqno(), hash}); + if (it == perm_states_.end()) { + promise.set_error(td::Status::Error(ErrorCode::notready)); return; } - promise.set_result(true); + promise.set_result(it->second.size); } void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, @@ -1023,15 +1032,15 @@ void ArchiveManager::persistent_state_gc(std::pair last) { int res = 0; BlockSeqno seqno = 0; - F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, - [&](const fileref::PersistentStateShort &x) { - res = 0; - seqno = x.masterchain_seqno; - }, - [&](const auto &obj) { res = -1; })); + F.id.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, + [&](const fileref::PersistentStateShort &x) { + res = 0; + seqno = x.masterchain_seqno; + }, + [&](const auto &obj) { res = -1; })); if (res == -1) { - td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + F.id.filename_short()).ignore(); perm_states_.erase(it); } if (res != 0) { @@ -1081,7 +1090,7 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pai CHECK(it != perm_states_.end()); auto &F = it->second; if (to_del) { - td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + F.id.filename_short()).ignore(); perm_states_.erase(it); } delay_action( @@ -1202,12 +1211,7 @@ void ArchiveManager::prepare_stats(td::Promise states; for (auto &[key, file] : perm_states_) { BlockSeqno seqno = key.first; - auto r_stat = td::stat(db_root_ + "/archive/states/" + file.filename_short()); - if (r_stat.is_error()) { - LOG(WARNING) << "Cannot stat persistent state file " << file.filename_short() << " : " << r_stat.move_as_error(); - } else { - states[seqno] += r_stat.move_as_ok().size_; - } + states[seqno] += file.size; } td::StringBuilder sb; for (auto &[seqno, size] : states) { @@ -1308,7 +1312,7 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han auto it = perm_states_.begin(); while (it != perm_states_.end()) { int res = 0; - it->second.ref().visit(td::overloaded( + it->second.id.ref().visit(td::overloaded( [&](const fileref::ZeroStateShort &x) { res = -1; }, [&](const fileref::PersistentStateShort &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, [&](const auto &obj) { res = 1; })); @@ -1317,7 +1321,7 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han } else { auto it2 = it; it++; - td::unlink(db_root_ + "/archive/states/" + it2->second.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + it2->second.id.filename_short()).ignore(); perm_states_.erase(it2); } } diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index d919e32e..cd79ccc6 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -52,7 +52,8 @@ class ArchiveManager : public td::actor::Actor { void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_size, td::Promise promise); - void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise); void check_zero_state(BlockIdExt block_id, td::Promise promise); void get_previous_persistent_state_files(BlockSeqno cur_mc_seqno, td::Promise>> promise); @@ -189,7 +190,11 @@ class ArchiveManager : public td::actor::Actor { return p.key ? key_files_ : p.temp ? temp_files_ : files_; } - std::map, FileReferenceShort> perm_states_; // Mc block seqno, hash -> state + struct PermState { + FileReferenceShort id; + td::uint64 size; + }; + std::map, PermState> perm_states_; // Mc block seqno, hash -> state void load_package(PackageId seqno); void delete_package(PackageId seqno, td::Promise promise); diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 8d83e7a7..2b370eb0 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -310,9 +310,9 @@ void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt mas offset, max_size, std::move(promise)); } -void RootDb::check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::check_persistent_state, block_id, masterchain_block_id, +void RootDb::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 52f6098e..9e5b0b30 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -80,8 +80,8 @@ class RootDb : public Db { td::Promise promise) override; void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) override; - void check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void get_zero_state_file(BlockIdExt block_id, td::Promise promise) override; void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) override; diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index da49f0e2..8dc2a681 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -275,20 +275,32 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise) { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() || !R.move_as_ok()) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); return; } - auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); }); auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_persistent_state_exists, block_id, + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, std::move(P)); +} + +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, masterchain_block_id, std::move(P)); } @@ -389,6 +401,21 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo ShardIdFull{masterchainId}, std::move(P)); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + ShardIdFull shard_prefix = create_shard_id(query.shard_prefix_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, + shard_prefix, std::move(P)); +} + void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise) { td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_, diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index ce0aedd3..d6160a94 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -66,6 +66,8 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, @@ -80,6 +82,8 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index ac0eb768..a8ddb338 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -509,13 +509,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise) { auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() || !R.move_as_ok()) { + td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); return; } - auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); }); @@ -523,7 +522,22 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod auto masterchain_block_id = create_block_id(query.masterchain_block_); VLOG(FULL_NODE_DEBUG) << "Got query preparePersistentState " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_persistent_state_exists, block_id, + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " + << masterchain_block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, masterchain_block_id, std::move(P)); } diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index fb3eef76..8cbd614a 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -123,6 +123,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 29ef715b..9a2eea56 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -62,8 +62,8 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) = 0; - virtual void check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; + virtual void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void get_zero_state_file(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 62fdc4b4..8e4a4d08 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -200,9 +200,9 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::check_persistent_state_file_exists, block_id, masterchain_block_id, +void ValidatorManagerImpl::get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index cd06bf55..62f54b2e 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -108,8 +108,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) override; void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 0b8b9e73..486c185f 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -128,8 +128,8 @@ class ValidatorManagerImpl : public ValidatorManager { void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override { UNREACHABLE(); } - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override { + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override { UNREACHABLE(); } void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager.cpp b/validator/manager.cpp index b0ac5409..b1dc4cc9 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -304,9 +304,9 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::check_persistent_state_file_exists, block_id, masterchain_block_id, +void ValidatorManagerImpl::get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager.hpp b/validator/manager.hpp index 418deb35..347b8f7e 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -377,8 +377,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) override; void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 6735a2b5..c481adbb 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -168,6 +168,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { }, [&, self = this](ton_api::tonNode_preparedState &f) { if (masterchain_block_id_.is_valid()) { + request_total_size(); got_block_state_part(td::BufferSlice{}, 0); return; } @@ -190,8 +191,37 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { create_serialize_tl_object_suffix(std::move(query)), td::Timestamp::in(3.0), std::move(P)); } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : download started"); })); - status_.set_status(PSTRING() << block_id_.id.to_str() << " : 0 bytes, 0B/s"); +} + +void DownloadState::request_total_size() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + return; + } + auto res = fetch_tl_object(R.move_as_ok(), true); + if (res.is_error()) { + return; + } + td::actor::send_closure(SelfId, &DownloadState::got_total_size, res.ok()->size_); + }); + + td::BufferSlice query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); + if (client_.empty()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, + "get size", std::move(P), td::Timestamp::in(3.0), std::move(query), + FullNode::max_state_size(), rldp_); + } else { + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get size", + create_serialize_tl_object_suffix(std::move(query)), + td::Timestamp::in(3.0), std::move(P)); + } +} + +void DownloadState::got_total_size(td::uint64 size) { + total_size_ = size; } void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 requested_size) { @@ -203,10 +233,22 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques if (elapsed > 5.0) { prev_logged_timer_ = td::Timer(); auto speed = (td::uint64)((double)(sum_ - prev_logged_sum_) / elapsed); - LOG(WARNING) << "downloading state " << block_id_.to_str() << ": " << td::format::as_size(sum_) << " (" - << td::format::as_size(speed) << "/s)"; - status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, " << td::format::as_size(speed) - << "/s"); + td::StringBuilder sb; + sb << td::format::as_size(sum_); + if (total_size_) { + sb << "/" << td::format::as_size(total_size_); + } + sb << " (" << td::format::as_size(speed) << "/s"; + if (total_size_) { + sb << ", " << td::StringBuilder::FixedDouble((double)sum_ / (double)total_size_ * 100.0, 2) << "%"; + if (speed > 0 && total_size_ >= sum_) { + td::uint64 rem = (total_size_ - sum_) / speed; + sb << ", " << rem << "s remaining"; + } + } + sb << ")"; + LOG(WARNING) << "downloading state " << block_id_.to_str() << " : " << sb.as_cslice(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sb.as_cslice()); prev_logged_sum_ = sum_; } diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index 470c5431..29854ce2 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -49,6 +49,8 @@ class DownloadState : public td::actor::Actor { void got_block_handle(BlockHandle handle); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_state_description(td::BufferSlice data_description); + void request_total_size(); + void got_total_size(td::uint64 size); void got_block_state_part(td::BufferSlice data, td::uint32 requested_size); void got_block_state(td::BufferSlice data); @@ -77,6 +79,7 @@ class DownloadState : public td::actor::Actor { td::uint64 prev_logged_sum_ = 0; td::Timer prev_logged_timer_; + td::uint64 total_size_ = 0; ProcessStatus status_; }; diff --git a/validator/validator.h b/validator/validator.h index 5d6c0173..eecb30c0 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -227,8 +227,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_data(BlockHandle handle, td::Promise promise) = 0; virtual void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_zero_state(BlockIdExt block_id, td::Promise promise) = 0; - virtual void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; + virtual void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) = 0; virtual void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,