From 037053fffec58dedf8e72051fb695a4a42d3d9f3 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 29 Apr 2024 16:33:10 +0300 Subject: [PATCH 1/7] More verbose state serializer logs (#976) Co-authored-by: SpyCheese --- crypto/vm/large-boc-serializer.cpp | 60 ++++++++++++++++++++++-------- validator/state-serializer.cpp | 30 +++++++++------ 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index fe16b767..fbd065dc 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -14,6 +14,9 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ +#include "td/utils/Time.h" +#include "td/utils/Timer.h" + #include #include "vm/boc.h" #include "vm/boc-writers.h" @@ -30,7 +33,8 @@ class LargeBocSerializer { public: using Hash = Cell::Hash; - explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) {} + explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) { + } void add_root(Hash root); td::Status import_cells(); @@ -65,7 +69,8 @@ class LargeBocSerializer { std::map cells; std::vector*> cell_list; struct RootInfo { - RootInfo(Hash hash, int idx) : hash(hash), idx(idx) {} + RootInfo(Hash hash, int idx) : hash(hash), idx(idx) { + } Hash hash; int idx; }; @@ -78,6 +83,10 @@ class LargeBocSerializer { void reorder_cells(); int revisit(int cell_idx, int force = 0); td::uint64 compute_sizes(int mode, int& r_size, int& o_size); + + td::Timestamp log_speed_at_; + size_t processed_cells_ = 0; + static constexpr double LOG_SPEED_PERIOD = 120.0; }; void LargeBocSerializer::add_root(Hash root) { @@ -85,12 +94,16 @@ void LargeBocSerializer::add_root(Hash root) { } td::Status LargeBocSerializer::import_cells() { + td::Timer timer; + log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); + processed_cells_ = 0; for (auto& root : roots) { TRY_RESULT(idx, import_cell(root.hash)); root.idx = idx; } reorder_cells(); CHECK(!cell_list.empty()); + LOG(ERROR) << "serializer: import_cells took " << timer.elapsed() << "s, " << cell_count << " cells"; return td::Status::OK(); } @@ -98,6 +111,12 @@ td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { if (depth > Cell::max_depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); } + ++processed_cells_; + if (log_speed_at_.is_in_past()) { + log_speed_at_ += LOG_SPEED_PERIOD; + LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; + processed_cells_ = 0; + } auto it = cells.find(hash); if (it != cells.end()) { it->second.should_cache = true; @@ -282,6 +301,7 @@ td::uint64 LargeBocSerializer::compute_sizes(int mode, int& r_size, int& o_size) } td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { + td::Timer timer; using Mode = BagOfCells::Mode; BagOfCells::Info info; if ((mode & Mode::WithCacheBits) && !(mode & Mode::WithIndex)) { @@ -313,13 +333,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { return td::Status::Error("bag of cells is too large"); } - boc_writers::FileWriter writer{fd, (size_t) info.total_size}; - auto store_ref = [&](unsigned long long value) { - writer.store_uint(value, info.ref_byte_size); - }; - auto store_offset = [&](unsigned long long value) { - writer.store_uint(value, info.offset_byte_size); - }; + boc_writers::FileWriter writer{fd, (size_t)info.total_size}; + auto store_ref = [&](unsigned long long value) { writer.store_uint(value, info.ref_byte_size); }; + auto store_offset = [&](unsigned long long value) { writer.store_uint(value, info.offset_byte_size); }; writer.store_uint(info.magic, 4); @@ -371,6 +387,8 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } DCHECK(writer.position() == info.data_offset); size_t keep_position = writer.position(); + log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); + processed_cells_ = 0; for (int i = 0; i < cell_count; ++i) { auto hash = cell_list[cell_count - 1 - i]->first; const auto& dc_info = cell_list[cell_count - 1 - i]->second; @@ -389,6 +407,12 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { DCHECK(k > i && k < cell_count); store_ref(k); } + ++processed_cells_; + if (log_speed_at_.is_in_past()) { + log_speed_at_ += LOG_SPEED_PERIOD; + LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; + processed_cells_ = 0; + } } DCHECK(writer.position() - keep_position == info.data_size); if (info.has_crc32c) { @@ -396,17 +420,23 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { writer.store_uint(td::bswap32(crc), 4); } DCHECK(writer.empty()); - return writer.finalize(); -} + TRY_STATUS(writer.finalize()); + LOG(ERROR) << "serializer: serialize took " << timer.elapsed() << "s, " << cell_count << " cells, " + << writer.position() << " bytes"; + return td::Status::OK(); } +} // namespace -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, - td::FileFd& fd, int mode) { +td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, + int mode) { + td::Timer timer; CHECK(reader != nullptr) LargeBocSerializer serializer(reader); serializer.add_root(root_hash); TRY_STATUS(serializer.import_cells()); - return serializer.serialize(fd, mode); + TRY_STATUS(serializer.serialize(fd, mode)); + LOG(ERROR) << "serialization took " << timer.elapsed() << "s"; + return td::Status::OK(); } -} +} // namespace vm diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 8e1b1b57..93363d3b 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -132,27 +132,33 @@ void AsyncStateSerializer::next_iteration() { if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { if (!have_masterchain_state_) { - LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().id; + LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str(); // block next attempts immediately, but send actual request later running_ = true; + double delay = td::Random::fast(0, 3600); + LOG(WARNING) << "serializer delay = " << delay << "s"; delay_action([SelfId = actor_id( this)]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_masterchain_state); }, - td::Timestamp::in(td::Random::fast(0, 3600))); + td::Timestamp::in(delay)); return; } while (next_idx_ < shards_.size()) { if (!need_monitor(shards_[next_idx_].shard_full())) { next_idx_++; } else { - // block next attempts immediately, but send actual request later - running_ = true; - delay_action( - [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, - td::Timestamp::in(td::Random::fast(0, 1800))); + // block next attempts immediately, but send actual request later + running_ = true; + double delay = td::Random::fast(0, 1800); + LOG(WARNING) << "serializer delay = " << delay << "s"; + delay_action( + [SelfId = actor_id(this), shard = shards_[next_idx_]]() { + td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); + }, + td::Timestamp::in(delay)); return; } } - LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().id; + LOG(ERROR) << "finished serializing persistent state for " << masterchain_handle_->id().id.to_str(); last_key_block_ts_ = masterchain_handle_->unix_time(); last_key_block_id_ = masterchain_handle_->id(); } @@ -194,7 +200,7 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { - LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().id; + LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str(); have_masterchain_state_ = true; CHECK(next_idx_ == 0); CHECK(shards_.size() == 0); @@ -217,7 +223,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state } void AsyncStateSerializer::stored_masterchain_state() { - LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().id; + LOG(ERROR) << "finished serializing masterchain state " << masterchain_handle_->id().id.to_str(); running_ = false; next_iteration(); } @@ -247,13 +253,13 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { - LOG(INFO) << "serializing shard state " << handle->id().id; + LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { R.ensure(); - LOG(INFO) << "finished serializing shard state " << handle->id().id; + LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), From 6fb2019a4fe16dcf298a46c9c719987b18411320 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 8 May 2024 17:41:15 +0300 Subject: [PATCH 2/7] Improve validator session stats (#982) * Add list of validators * Fix producer stats * Make round and stats timestamps more meaningful Co-authored-by: SpyCheese --- tl/generate/scheme/ton_api.tl | 8 +++- tl/generate/scheme/ton_api.tlo | Bin 88712 -> 89348 bytes validator-session/validator-session-types.h | 18 ++++++++- validator-session/validator-session.cpp | 39 ++++++++++++-------- validator/interfaces/validator-manager.h | 1 + validator/manager-disk.hpp | 3 ++ validator/manager-hardfork.hpp | 3 ++ validator/manager.cpp | 28 +++++++++++++- validator/manager.hpp | 1 + validator/validator-group.cpp | 19 ++++++++++ 10 files changed, 99 insertions(+), 21 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index af9df3fd..3db2ace9 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -761,13 +761,17 @@ validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int co signed_weight:long signed_33pct_at:double signed_66pct_at:double serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; -validatorSession.statsRound timestamp:long producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; +validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; -validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:long self:int256 session_id:int256 cc_seqno:int +validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self:int256 session_id:int256 cc_seqno:int creator:int256 total_validators:int total_weight:long signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; +validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; +validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int timestamp:double + self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; + ---functions--- ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index c97fa2feefc0767775d31b5c072aaf671c979e35..b0d5f315b0603c43be7a61d0f9290c55a69a744a 100644 GIT binary patch delta 461 zcmeC!%G$DvmG{wXeJchiVBN^;Y^c8fyu=yxvc#Osl*E$!qTtlx;>`R!J-^iQFqoKo zQGRK`VABHA6KaB6_>R_0Vhi*b}Nn%OyP$9ux9mpKfT$C_VX1fWTynZ!VKpd{v#kmWxq%@`+gj zoE*ieIcf2kDHW49T1Zchd8Z-)avdYo$3@O;Ivlob2SFxHzro3v!wT|^*z|(}jAE<| zAolbWF2)#;N!u@QF<{9 delta 183 zcmZqK#oDozmG{wXeJchiVBW~sNXEq%U z+qQ!s9m|*SF|t5Ro^Hs)C^mh+5F;xmNcGKQLOx8>dwCe8Cs(8jO#dOoXu|>$-+o4z H(ZUk|G!Q%& diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index db06bbd9..e13c36d2 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -129,7 +129,7 @@ struct ValidatorSessionStats { } }; struct Round { - td::uint64 timestamp = 0; + double timestamp = -1.0; std::vector producers; }; @@ -139,7 +139,7 @@ struct ValidatorSessionStats { bool success = false; ValidatorSessionId session_id = ValidatorSessionId::zero(); CatchainSeqno cc_seqno = 0; - td::uint64 timestamp = 0; + double timestamp = -1.0; PublicKeyHash self = PublicKeyHash::zero(); PublicKeyHash creator = PublicKeyHash::zero(); td::uint32 total_validators = 0; @@ -150,6 +150,20 @@ struct ValidatorSessionStats { ValidatorWeight approve_signatures_weight = 0; }; +struct NewValidatorGroupStats { + struct Node { + PublicKeyHash id = PublicKeyHash::zero(); + ValidatorWeight weight = 0; + }; + + ValidatorSessionId session_id = ValidatorSessionId::zero(); + ShardIdFull shard{masterchainId}; + CatchainSeqno cc_seqno = 0; + double timestamp = -1.0; + td::uint32 self_idx = 0; + std::vector nodes; +}; + } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 215b6b12..d2fe6458 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -871,7 +871,7 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { callback_->on_block_skipped(cur_round_); } else { cur_stats_.success = true; - cur_stats_.timestamp = (td::uint64)td::Clocks::system(); + cur_stats_.timestamp = td::Clocks::system(); cur_stats_.signatures = (td::uint32)export_sigs.size(); cur_stats_.signatures_weight = signatures_weight; cur_stats_.approve_signatures = (td::uint32)export_approve_sigs.size(); @@ -899,6 +899,12 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { cur_round_++; if (have_block) { stats_init(); + } else { + size_t round_idx = cur_round_ - cur_stats_.first_round; + while (round_idx >= cur_stats_.rounds.size()) { + stats_add_round(); + } + cur_stats_.rounds[round_idx].timestamp = td::Clocks::system(); } auto it2 = blocks_.begin(); while (it2 != blocks_.end()) { @@ -988,9 +994,7 @@ void ValidatorSessionImpl::destroy() { } void ValidatorSessionImpl::get_current_stats(td::Promise promise) { - ValidatorSessionStats stats = cur_stats_; - stats.timestamp = (td::uint64)td::Clocks::system(); - promise.set_result(std::move(stats)); + promise.set_result(cur_stats_); } void ValidatorSessionImpl::get_validator_group_info_for_litequery( @@ -1084,26 +1088,31 @@ void ValidatorSessionImpl::stats_init() { ++it; } } + + if (cur_stats_.rounds.empty()) { + stats_add_round(); + } + cur_stats_.rounds[0].timestamp = td::Clocks::system(); stats_inited_ = true; } void ValidatorSessionImpl::stats_add_round() { + td::uint32 round = cur_stats_.first_round + cur_stats_.rounds.size(); cur_stats_.rounds.emplace_back(); - auto& round = cur_stats_.rounds.back(); - round.timestamp = (td::uint64)td::Clocks::system(); - round.producers.resize(description().get_max_priority() + 1); + auto& stat = cur_stats_.rounds.back(); + stat.producers.resize(description().get_max_priority() + 1); for (td::uint32 i = 0; i < description().get_total_nodes(); i++) { - td::int32 priority = description().get_node_priority(i, cur_round_); + td::int32 priority = description().get_node_priority(i, round); if (priority >= 0) { - CHECK((size_t)priority < round.producers.size()); - round.producers[priority].id = description().get_source_id(i); - round.producers[priority].is_ours = (local_idx() == i); - round.producers[priority].approvers.resize(description().get_total_nodes(), false); - round.producers[priority].signers.resize(description().get_total_nodes(), false); + CHECK((size_t)priority < stat.producers.size()); + stat.producers[priority].id = description().get_source_id(i); + stat.producers[priority].is_ours = (local_idx() == i); + stat.producers[priority].approvers.resize(description().get_total_nodes(), false); + stat.producers[priority].signers.resize(description().get_total_nodes(), false); } } - while (!round.producers.empty() && round.producers.back().id.is_zero()) { - round.producers.pop_back(); + while (!stat.producers.empty() && stat.producers.back().id.is_zero()) { + stat.producers.pop_back(); } } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 41412fb1..278915f1 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -171,6 +171,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) = 0; virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; + virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index d5a6e909..eca6a741 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -378,6 +378,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 7937729c..f0cf83de 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -439,6 +439,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager.cpp b/validator/manager.cpp index 37281cc9..ddc58225 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2729,7 +2729,7 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, stats.cc_seqno, stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures, stats.signatures_weight, stats.approve_signatures, stats.approve_signatures_weight, stats.first_round, std::move(rounds)); - std::string s = td::json_encode(td::ToJson(*obj.get()), false); + auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); std::ofstream file; @@ -2737,7 +2737,31 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, file << s << "\n"; file.close(); - LOG(INFO) << "Writing validator session stats for " << block_id.id; + LOG(INFO) << "Writing validator session stats for " << block_id.id.to_str(); +} + +void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + std::vector> nodes; + for (const auto &node : stats.nodes) { + nodes.push_back( + create_tl_object(node.id.bits256_value(), node.weight)); + } + auto obj = create_tl_object( + stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.timestamp, stats.self_idx, + std::move(nodes)); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + + LOG(INFO) << "Writing new validator group stats for " << stats.shard.to_str(); } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 7e5930d3..d6d0307b 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -565,6 +565,7 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 5193e523..6a71d306 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -377,6 +377,22 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch prev_block_ids_ = std::vector{next_block_id}; } postponed_accept_.clear(); + + validatorsession::NewValidatorGroupStats stats; + stats.session_id = session_id_; + stats.shard = shard_; + stats.cc_seqno = validator_set_->get_catchain_seqno(); + stats.timestamp = td::Clocks::system(); + td::uint32 idx = 0; + for (const auto& node : validator_set_->export_vector()) { + PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); + if (id == local_id_) { + stats.self_idx = idx; + } + stats.nodes.push_back(validatorsession::NewValidatorGroupStats::Node{id, node.weight}); + ++idx; + } + td::actor::send_closure(manager_, &ValidatorManager::log_new_validator_group_stats, std::move(stats)); } void ValidatorGroup::destroy() { @@ -390,6 +406,9 @@ void ValidatorGroup::destroy() { return; } auto stats = R.move_as_ok(); + if (stats.rounds.empty()) { + return; + } stats.cc_seqno = cc_seqno; td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, std::move(stats)); From d5c09936cf019f57d5a3bb7246bc5445006db8ae Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 10 May 2024 17:04:49 +0300 Subject: [PATCH 3/7] Block broadcasts in custom overlays (#986) Co-authored-by: SpyCheese --- create-hardfork/create-hardfork.cpp | 2 +- test/test-ton-collator.cpp | 2 +- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 89348 -> 89388 bytes .../validator-engine-console-query.cpp | 7 +- validator-engine/validator-engine.cpp | 22 +-- validator/full-node-private-overlay.cpp | 79 ++++++++--- validator/full-node-private-overlay.hpp | 34 +++-- validator/full-node-shard.cpp | 21 +-- validator/full-node-shard.h | 2 +- validator/full-node-shard.hpp | 3 +- validator/full-node.cpp | 130 ++++++++++++------ validator/full-node.h | 17 ++- validator/full-node.hpp | 25 ++-- validator/impl/accept-block.cpp | 8 +- validator/interfaces/validator-manager.h | 2 +- validator/manager-disk.hpp | 2 +- validator/manager-hardfork.hpp | 2 +- validator/manager.cpp | 4 +- validator/manager.hpp | 2 +- validator/validator.h | 2 +- 21 files changed, 234 insertions(+), 134 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index c5f1add8..a9e7403a 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -246,7 +246,7 @@ class HardforkCreator : public td::actor::Actor { } void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override { } - void send_broadcast(ton::BlockBroadcast broadcast) override { + void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 286a5fea..d36db5d2 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -347,7 +347,7 @@ class TestNode : public td::actor::Actor { } } } - void send_broadcast(ton::BlockBroadcast broadcast) override { + void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 3db2ace9..d0a063f7 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -595,7 +595,7 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) gc:engine.gc = engine.validator.Config; -engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int = engine.validator.CustomOverlayNode; +engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int block_sender:Bool = engine.validator.CustomOverlayNode; engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index b0d5f315b0603c43be7a61d0f9290c55a69a744a..fb7f219765fea2ff1aa4dee29f67166b388d01f9 100644 GIT binary patch delta 91 zcmZqK#kyt}>jnW4*8lh1HI*jc6qeg;EMnTg0^&~2nW6w;Oqim=2@<(^Ovs02^29mn mlb=lSnY_S~l_x1DKRG+TI5jUNwP^Cge0i|?%{Eg{90vgF<0sYt delta 78 zcmZ3pi?wAJ>jnW4)<<2JLzE`p6qeg;EMnTg0^&~2nW6w;Oqim=2@<(^Ovs0Ma^p(z Z$rt7bOnx%OXL5oS2UyAGl&L3<0{~H}Bq9I+ diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 956c23aa..38f524fc 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1171,9 +1171,10 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "Overlay \"" << overlay->name_ << "\": " << overlay->nodes_.size() << " nodes\n"; for (const auto &node : overlay->nodes_) { td::TerminalIO::out() << " " << node->adnl_id_ - << (node->msg_sender_ ? (PSTRING() << " (sender, p=" << node->msg_sender_priority_ << ")") - : "") - << "\n"; + << (node->msg_sender_ + ? (PSTRING() << " (msg sender, p=" << node->msg_sender_priority_ << ")") + : "") + << (node->block_sender_ ? " (block sender)" : "") << "\n"; } td::TerminalIO::out() << "\n"; } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8ba99178..3c98a4ac 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2357,16 +2357,9 @@ void ValidatorEngine::load_custom_overlays_config() { } for (auto &overlay : custom_overlays_config_->overlays_) { - std::vector nodes; - std::map senders; - for (const auto &node : overlay->nodes_) { - nodes.emplace_back(node->adnl_id_); - if (node->msg_sender_) { - senders[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; - } - } - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_ext_msg_overlay, std::move(nodes), - std::move(senders), overlay->name_, [](td::Result R) { R.ensure(); }); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_custom_overlay, + ton::validator::fullnode::CustomOverlayParams::fetch(*overlay), + [](td::Result R) { R.ensure(); }); } } @@ -3571,11 +3564,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustom senders[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; } } - std::string name = overlay->name_; + auto params = ton::validator::fullnode::CustomOverlayParams::fetch(*query.overlay_); td::actor::send_closure( - full_node_, &ton::validator::fullnode::FullNode::add_ext_msg_overlay, std::move(nodes), std::move(senders), - std::move(name), - [SelfId = actor_id(this), overlay = std::move(overlay), + full_node_, &ton::validator::fullnode::FullNode::add_custom_overlay, std::move(params), + [SelfId = actor_id(this), overlay = std::move(query.overlay_), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); @@ -3605,7 +3597,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustom return; } td::actor::send_closure( - full_node_, &ton::validator::fullnode::FullNode::del_ext_msg_overlay, query.name_, + full_node_, &ton::validator::fullnode::FullNode::del_custom_overlay, query.name_, [SelfId = actor_id(this), name = query.name_, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index a64c0e9b..74bb75c5 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -38,17 +38,7 @@ void FullNodePrivateBlockOverlay::process_block_broadcast(PublicKeyHash src, ton } VLOG(FULL_NODE_DEBUG) << "Received block broadcast in private overlay from " << src << ": " << B.ok().block_id.to_str(); - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - if (R.error().code() == ErrorCode::notready) { - LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); - } else { - LOG(INFO) << "dropped broadcast: " << R.move_as_error(); - } - } - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(), - std::move(P)); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); } void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { @@ -60,6 +50,9 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api:: } void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (adnl::AdnlNodeIdShort{src} == local_id_) { + return; + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return; @@ -169,18 +162,47 @@ void FullNodePrivateBlockOverlay::tear_down() { } } -void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query) { - auto it = senders_.find(adnl::AdnlNodeIdShort{src}); - if (it == senders_.end()) { +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { + VLOG(FULL_NODE_DEBUG) << "Dropping block broadcast in private overlay \"" << name_ << "\" from unauthorized sender " + << src; return; } - LOG(FULL_NODE_DEBUG) << "Got external message in private overlay \"" << name_ << "\" from " << src - << " (priority=" << it->second << ")"; + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Received block broadcast in custom overlay \"" << name_ << "\" from " << src << ": " + << B.ok().block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query) { + auto it = msg_senders_.find(adnl::AdnlNodeIdShort{src}); + if (it == msg_senders_.end()) { + VLOG(FULL_NODE_DEBUG) << "Dropping external message broadcast in custom overlay \"" << name_ + << "\" from unauthorized sender " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Got external message in custom overlay \"" << name_ << "\" from " << src + << " (priority=" << it->second << ")"; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_external_message, std::move(query.message_->data_), it->second); } void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (adnl::AdnlNodeIdShort{src} == local_id_) { + return; + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return; @@ -192,7 +214,7 @@ void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) { if (!inited_ || config_.ext_messages_broadcast_disabled_) { return; } - LOG(FULL_NODE_DEBUG) << "Sending external message to private overlay \"" << name_ << "\""; + VLOG(FULL_NODE_DEBUG) << "Sending external message to custom overlay \"" << name_ << "\""; auto B = create_serialize_tl_object( create_tl_object(std::move(data))); if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { @@ -204,6 +226,21 @@ void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) { } } +void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending block broadcast to custom overlay \"" << name_ + << "\": " << broadcast.block_id.to_str(); + auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; + } + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + void FullNodeCustomOverlay::start_up() { std::sort(nodes_.begin(), nodes_.end()); nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); @@ -234,7 +271,8 @@ void FullNodeCustomOverlay::try_init() { void FullNodeCustomOverlay::init() { LOG(FULL_NODE_WARNING) << "Creating custom overlay \"" << name_ << "\" for adnl id " << local_id_ << " : " - << nodes_.size() << " nodes, overlay_id=" << overlay_id_; + << nodes_.size() << " nodes, " << msg_senders_.size() << " msg senders, " + << block_senders_.size() << " block senders, overlay_id=" << overlay_id_; class Callback : public overlay::Overlays::Callback { public: void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { @@ -256,9 +294,12 @@ void FullNodeCustomOverlay::init() { }; std::map authorized_keys; - for (const auto &sender : senders_) { + for (const auto &sender : msg_senders_) { authorized_keys[sender.first.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); } + for (const auto &sender : block_senders_) { + authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); + } overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; td::actor::send_closure( overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), nodes_, diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index c651acef..196d6da6 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -52,7 +52,8 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, - td::actor::ActorId validator_manager) + td::actor::ActorId validator_manager, + td::actor::ActorId full_node) : local_id_(local_id) , nodes_(std::move(nodes)) , zero_state_file_hash_(zero_state_file_hash) @@ -63,7 +64,8 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { , rldp_(rldp) , rldp2_(rldp2) , overlays_(overlays) - , validator_manager_(validator_manager) { + , validator_manager_(validator_manager) + , full_node_(full_node) { } private: @@ -79,6 +81,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; + td::actor::ActorId full_node_; bool inited_ = false; overlay::OverlayIdFull overlay_id_full_; @@ -90,6 +93,10 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { class FullNodeCustomOverlay : public td::actor::Actor { public: + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { @@ -98,6 +105,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void send_external_message(td::BufferSlice data); + void send_broadcast(BlockBroadcast broadcast); void set_config(FullNodeConfig config) { config_ = std::move(config); @@ -106,16 +114,17 @@ class FullNodeCustomOverlay : public td::actor::Actor { void start_up() override; void tear_down() override; - FullNodeCustomOverlay(adnl::AdnlNodeIdShort local_id, std::vector nodes, - std::map senders, std::string name, FileHash zero_state_file_hash, + FullNodeCustomOverlay(adnl::AdnlNodeIdShort local_id, CustomOverlayParams params, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, - td::actor::ActorId validator_manager) + td::actor::ActorId validator_manager, + td::actor::ActorId full_node) : local_id_(local_id) - , nodes_(std::move(nodes)) - , senders_(std::move(senders)) - , name_(std::move(name)) + , name_(std::move(params.name_)) + , nodes_(std::move(params.nodes_)) + , msg_senders_(std::move(params.msg_senders_)) + , block_senders_(std::move(params.block_senders_)) , zero_state_file_hash_(zero_state_file_hash) , config_(config) , keyring_(keyring) @@ -123,14 +132,16 @@ class FullNodeCustomOverlay : public td::actor::Actor { , rldp_(rldp) , rldp2_(rldp2) , overlays_(overlays) - , validator_manager_(validator_manager) { + , validator_manager_(validator_manager) + , full_node_(full_node) { } private: adnl::AdnlNodeIdShort local_id_; - std::vector nodes_; - std::map senders_; std::string name_; + std::vector nodes_; + std::map msg_senders_; + std::set block_senders_; FileHash zero_state_file_hash_; FullNodeConfig config_; @@ -140,6 +151,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; + td::actor::ActorId full_node_; bool inited_ = false; overlay::OverlayIdFull overlay_id_full_; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 433d9469..6e265cbc 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -661,17 +661,7 @@ void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonN return; } VLOG(FULL_NODE_DEBUG) << "Received block broadcast from " << src << ": " << B.ok().block_id.to_str(); - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - if (R.error().code() == ErrorCode::notready) { - LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); - } else { - LOG(INFO) << "dropped broadcast: " << R.move_as_error(); - } - } - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(), - std::move(P)); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { @@ -1137,7 +1127,8 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client) + td::actor::ActorId client, + td::actor::ActorId full_node) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1149,6 +1140,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , overlays_(overlays) , validator_manager_(validator_manager) , client_(client) + , full_node_(full_node) , config_(config) { } @@ -1157,9 +1149,10 @@ td::actor::ActorOwn FullNodeShard::create( FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client) { + td::actor::ActorId client, td::actor::ActorId full_node) { return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, config, - keyring, adnl, rldp, rldp2, overlays, validator_manager, client); + keyring, adnl, rldp, rldp2, overlays, validator_manager, client, + full_node); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 1b742fb9..2ed13d66 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -72,7 +72,7 @@ class FullNodeShard : public td::actor::Actor { FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client); + td::actor::ActorId client, td::actor::ActorId full_node); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index c8b5301a..1fdbce1c 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -214,7 +214,7 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client); + td::actor::ActorId client, td::actor::ActorId full_node); private: bool use_new_download() const { @@ -236,6 +236,7 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; td::actor::ActorId client_; + td::actor::ActorId full_node_; td::uint32 attempt_ = 0; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index affb8bcc..8d0f26bc 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -36,8 +36,8 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr local_keys_.insert(key); create_private_block_overlay(key); - for (auto &p : private_custom_overlays_) { - update_ext_msg_overlay(p.first, p.second); + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); } if (!sign_cert_by_.is_zero()) { @@ -64,8 +64,8 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr } local_keys_.erase(key); private_block_overlays_.erase(key); - for (auto &p : private_custom_overlays_) { - update_ext_msg_overlay(p.first, p.second); + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); } if (sign_cert_by_ != key) { @@ -119,8 +119,8 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise nodes, - std::map senders, std::string name, - td::Promise promise) { - if (nodes.empty()) { +void FullNodeImpl::add_custom_overlay(CustomOverlayParams params, td::Promise promise) { + if (params.nodes_.empty()) { promise.set_error(td::Status::Error("list of nodes is empty")); return; } - if (private_custom_overlays_.count(name)) { - promise.set_error(td::Status::Error(PSTRING() << "duplicate overlay name \"" << name << "\"")); + std::string name = params.name_; + if (custom_overlays_.count(name)) { + promise.set_error(td::Status::Error(PSTRING() << "duplicate custom overlay name \"" << name << "\"")); return; } - VLOG(FULL_NODE_WARNING) << "Adding private overlay for external messages \"" << name << "\", " << nodes.size() - << " nodes"; - auto &p = private_custom_overlays_[name]; - p.nodes_ = nodes; - p.senders_ = senders; - update_ext_msg_overlay(name, p); + VLOG(FULL_NODE_WARNING) << "Adding custom overlay \"" << name << "\", " << params.nodes_.size() << " nodes"; + auto &p = custom_overlays_[name]; + p.params_ = std::move(params); + update_custom_overlay(p); promise.set_result(td::Unit()); } -void FullNodeImpl::del_ext_msg_overlay(std::string name, td::Promise promise) { - auto it = private_custom_overlays_.find(name); - if (it == private_custom_overlays_.end()) { +void FullNodeImpl::del_custom_overlay(std::string name, td::Promise promise) { + auto it = custom_overlays_.find(name); + if (it == custom_overlays_.end()) { promise.set_error(td::Status::Error(PSTRING() << "no such overlay \"" << name << "\"")); return; } - private_custom_overlays_.erase(it); + custom_overlays_.erase(it); promise.set_result(td::Unit()); } @@ -182,8 +179,9 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { void FullNodeImpl::add_shard(ShardIdFull shard) { while (true) { if (shards_.count(shard) == 0) { - shards_.emplace(shard, FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, - adnl_, rldp_, rldp2_, overlays_, validator_manager_, client_)); + shards_.emplace(shard, + FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, adnl_, + rldp_, rldp2_, overlays_, validator_manager_, client_, actor_id(this))); if (all_validators_.size() > 0) { td::actor::send_closure(shards_[shard], &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } @@ -221,10 +219,10 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; return; } - for (auto &private_overlay : private_custom_overlays_) { + for (auto &private_overlay : custom_overlays_) { for (auto &actor : private_overlay.second.actors_) { auto local_id = actor.first; - if (private_overlay.second.senders_.count(local_id)) { + if (private_overlay.second.params_.msg_senders_.count(local_id)) { td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_external_message, data.clone()); } } @@ -245,7 +243,11 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); } -void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { +void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) { + send_block_broadcast_to_custom_overlays(broadcast); + if (custom_overlays_only) { + return; + } auto shard = get_shard(ShardIdFull{masterchainId}); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; @@ -404,7 +406,22 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { std::move(P)); } +void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { + send_block_broadcast_to_custom_overlays(broadcast); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(broadcast), + [](td::Result R) { + if (R.is_error()) { + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); + } else { + LOG(INFO) << "dropped broadcast: " << R.move_as_error(); + } + } + }); +} + void FullNodeImpl::start_up() { + add_shard(ShardIdFull{masterchainId}); if (local_id_.is_zero()) { if(adnl_id_.is_zero()) { auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; @@ -435,8 +452,8 @@ void FullNodeImpl::start_up() { void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_shard_block_info, block_id, cc_seqno, std::move(data)); } - void send_broadcast(BlockBroadcast broadcast) override { - td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast)); + void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { + td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), custom_overlays_only); } void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { @@ -488,8 +505,8 @@ void FullNodeImpl::start_up() { } void FullNodeImpl::update_private_overlays() { - for (auto &p : private_custom_overlays_) { - update_ext_msg_overlay(p.first, p.second); + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); } private_block_overlays_.clear(); @@ -520,28 +537,30 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { } private_block_overlays_[key] = td::actor::create_actor( "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, - private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_); + private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, + actor_id(this)); } } -void FullNodeImpl::update_ext_msg_overlay(const std::string &name, ExtMsgOverlayInfo &overlay) { +void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { auto old_actors = std::move(overlay.actors_); overlay.actors_.clear(); + CustomOverlayParams ¶ms = overlay.params_; auto try_local_id = [&](const adnl::AdnlNodeIdShort &local_id) { - if (std::find(overlay.nodes_.begin(), overlay.nodes_.end(), local_id) != overlay.nodes_.end()) { + if (std::find(params.nodes_.begin(), params.nodes_.end(), local_id) != params.nodes_.end()) { auto it = old_actors.find(local_id); if (it != old_actors.end()) { overlay.actors_[local_id] = std::move(it->second); old_actors.erase(it); } else { overlay.actors_[local_id] = td::actor::create_actor( - "CustomOverlay", local_id, overlay.nodes_, overlay.senders_, name, zero_state_file_hash_, config_, - keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_); + "CustomOverlay", local_id, params, zero_state_file_hash_, config_, keyring_, adnl_, rldp_, rldp2_, + overlays_, validator_manager_, actor_id(this)); } } }; try_local_id(adnl_id_); - for (const PublicKeyHash &local_key: local_keys_) { + for (const PublicKeyHash &local_key : local_keys_) { auto it = current_validators_.find(local_key); if (it != current_validators_.end()) { try_local_id(it->second); @@ -549,6 +568,25 @@ void FullNodeImpl::update_ext_msg_overlay(const std::string &name, ExtMsgOverlay } } +void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast) { + if (!custom_overlays_sent_broadcasts_.insert(broadcast.block_id).second) { + return; + } + custom_overlays_sent_broadcasts_lru_.push(broadcast.block_id); + if (custom_overlays_sent_broadcasts_lru_.size() > 256) { + custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); + custom_overlays_sent_broadcasts_lru_.pop(); + } + for (auto &private_overlay : custom_overlays_) { + for (auto &actor : private_overlay.second.actors_) { + auto local_id = actor.first; + if (private_overlay.second.params_.block_senders_.count(local_id)) { + td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_broadcast, broadcast.clone()); + } + } + } +} + FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, @@ -569,7 +607,6 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , client_(client) , db_root_(db_root) , config_(config) { - add_shard(ShardIdFull{masterchainId}); } td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, @@ -598,6 +635,21 @@ bool FullNodeConfig::operator!=(const FullNodeConfig &rhs) const { return !(*this == rhs); } +CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_customOverlay& f) { + CustomOverlayParams c; + c.name_ = f.name_; + for (const auto &node : f.nodes_) { + c.nodes_.emplace_back(node->adnl_id_); + if (node->msg_sender_) { + c.msg_senders_[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; + } + if (node->block_sender_) { + c.block_senders_.emplace(node->adnl_id_); + } + } + return c; +} + } // namespace fullnode } // namespace validator diff --git a/validator/full-node.h b/validator/full-node.h index 82e0dd5c..ac260dd7 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -55,6 +55,15 @@ struct FullNodeConfig { bool ext_messages_broadcast_disabled_ = false; }; +struct CustomOverlayParams { + std::string name_; + std::vector nodes_; + std::map msg_senders_; + std::set block_senders_; + + static CustomOverlayParams fetch(const ton_api::engine_validator_customOverlay& f); +}; + class FullNode : public td::actor::Actor { public: virtual ~FullNode() = default; @@ -74,10 +83,10 @@ class FullNode : public td::actor::Actor { virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; virtual void set_config(FullNodeConfig config) = 0; - virtual void add_ext_msg_overlay(std::vector nodes, - std::map senders, std::string name, - td::Promise promise) = 0; - virtual void del_ext_msg_overlay(std::string name, td::Promise promise) = 0; + virtual void add_custom_overlay(CustomOverlayParams params, td::Promise promise) = 0; + virtual void del_custom_overlay(std::string name, td::Promise promise) = 0; + + virtual void process_block_broadcast(BlockBroadcast broadcast) = 0; static constexpr td::uint32 max_block_size() { return 4 << 20; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 86664a3b..946a1705 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -27,6 +27,7 @@ #include #include +#include namespace ton { @@ -53,9 +54,8 @@ class FullNodeImpl : public FullNode { void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; void set_config(FullNodeConfig config) override; - void add_ext_msg_overlay(std::vector nodes, std::map senders, - std::string name, td::Promise promise) override; - void del_ext_msg_overlay(std::string name, td::Promise promise) override; + void add_custom_overlay(CustomOverlayParams params, td::Promise promise) override; + void del_custom_overlay(std::string name, td::Promise promise) override; void add_shard(ShardIdFull shard); void del_shard(ShardIdFull shard); @@ -66,7 +66,7 @@ class FullNodeImpl : public FullNode { void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data); - void send_broadcast(BlockBroadcast broadcast); + void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); @@ -83,6 +83,8 @@ class FullNodeImpl : public FullNode { void got_key_block_state(td::Ref state); void new_key_block(BlockHandle handle); + void process_block_broadcast(BlockBroadcast broadcast) override; + void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, @@ -123,18 +125,19 @@ class FullNodeImpl : public FullNode { std::map> private_block_overlays_; bool private_block_overlays_enable_compression_ = false; - struct ExtMsgOverlayInfo { - std::vector nodes_; - std::map senders_; - std::map> - actors_; // our local id -> actor + struct CustomOverlayInfo { + CustomOverlayParams params_; + std::map> actors_; // our local id -> actor }; - std::map private_custom_overlays_; + std::map custom_overlays_; + std::set custom_overlays_sent_broadcasts_; + std::queue custom_overlays_sent_broadcasts_lru_; void update_private_overlays(); void set_private_block_overlays_enable_compression(bool value); void create_private_block_overlay(PublicKeyHash key); - void update_ext_msg_overlay(const std::string& name, ExtMsgOverlayInfo& overlay); + void update_custom_overlay(CustomOverlayInfo& overlay); + void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); }; } // namespace fullnode diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index cda1c787..3da1167a 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -899,11 +899,6 @@ void AcceptBlockQuery::written_block_info_2() { } void AcceptBlockQuery::applied() { - if (!send_broadcast_) { - finish_query(); - return; - } - BlockBroadcast b; b.data = data_->data(); b.block_id = id_; @@ -923,7 +918,8 @@ void AcceptBlockQuery::applied() { } // do not wait for answer - td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b)); + td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b), + /* custom_overlays_only = */ !send_broadcast_); finish_query(); } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 278915f1..d435b756 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -133,7 +133,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_external_message(td::Ref message) = 0; virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; - virtual void send_block_broadcast(BlockBroadcast broadcast) = 0; + virtual void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index eca6a741..5cf93884 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -251,7 +251,7 @@ class ValidatorManagerImpl : public ValidatorManager { new_ihr_message(message->serialize()); } void send_top_shard_block_description(td::Ref desc) override; - void send_block_broadcast(BlockBroadcast broadcast) override { + void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index f0cf83de..fb5237d6 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -317,7 +317,7 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override { UNREACHABLE(); } - void send_block_broadcast(BlockBroadcast broadcast) override { + void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index ddc58225..82cc10c2 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1503,8 +1503,8 @@ void ValidatorManagerImpl::send_top_shard_block_description(td::Refsend_broadcast(std::move(broadcast)); +void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) { + callback_->send_broadcast(std::move(broadcast), custom_overlays_only); } void ValidatorManagerImpl::start_up() { diff --git a/validator/manager.hpp b/validator/manager.hpp index d6d0307b..65636a94 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -473,7 +473,7 @@ class ValidatorManagerImpl : public ValidatorManager { void send_external_message(td::Ref message) override; void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; - void send_block_broadcast(BlockBroadcast broadcast) override; + void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; diff --git a/validator/validator.h b/validator/validator.h index 9ede0711..17e55c70 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -134,7 +134,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; - virtual void send_broadcast(BlockBroadcast broadcast) = 0; + virtual void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only = false) = 0; virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, From 136b99b8d10a0219a9616ac6e7fa1e3be2338c6c Mon Sep 17 00:00:00 2001 From: neodix42 Date: Sun, 12 May 2024 04:48:14 -0500 Subject: [PATCH 4/7] Use jemalloc (#987) * use jemalloc * install system jemalloc in github action * check if jemalloc is actually used --------- Co-authored-by: neodiX --- .github/workflows/build-ton-linux-x86-64-shared.yml | 2 +- CMake/{FindJeMalloc.cmake => Findjemalloc.cmake} | 0 CMakeLists.txt | 2 +- Dockerfile | 6 +++--- assembly/native/build-ubuntu-shared.sh | 6 ++++-- 5 files changed, 9 insertions(+), 7 deletions(-) rename CMake/{FindJeMalloc.cmake => Findjemalloc.cmake} (100%) diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index ce0ade64..34f92d93 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -19,7 +19,7 @@ jobs: - name: Install system libraries run: | sudo apt-get update - sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev - name: Install clang-16 run: | diff --git a/CMake/FindJeMalloc.cmake b/CMake/Findjemalloc.cmake similarity index 100% rename from CMake/FindJeMalloc.cmake rename to CMake/Findjemalloc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 89be3238..658eab70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,7 +234,7 @@ if (THREADS_HAVE_PTHREAD_ARG) endif() if (TON_USE_JEMALLOC) - find_package(JeMalloc REQUIRED) + find_package(jemalloc REQUIRED) endif() set(MEMPROF "" CACHE STRING "Use one of \"ON\", \"FAST\" or \"SAFE\" to enable memory profiling. \ diff --git a/Dockerfile b/Dockerfile index 76c06b35..c5120e83 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:22.04 as builder RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev && \ rm -rf /var/lib/apt/lists/* ENV CC clang ENV CXX clang++ @@ -14,12 +14,12 @@ COPY ./ ./ RUN mkdir build && \ cd build && \ - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= .. && \ + cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \ ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client FROM ubuntu:22.04 RUN apt-get update && \ - apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev && \ + apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev && \ rm -rf /var/lib/apt/lists/* RUN mkdir -p /var/ton-work/db && \ diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 7d5326a4..ec868ecd 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -1,7 +1,7 @@ #/bin/bash #sudo apt-get update -#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev +#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev with_tests=false with_artifacts=false @@ -42,7 +42,7 @@ else echo "Using compiled openssl_3" fi -cmake -GNinja .. \ +cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_ROOT_DIR=$opensslPath \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ @@ -96,6 +96,8 @@ test $? -eq 0 || { echo "Can't strip final binaries"; exit 1; } ./lite-client/lite-client -V || exit 1 ./crypto/fift -V || exit 1 +ldd ./validator-engine/validator-engine || exit 1 + cd .. if [ "$with_artifacts" = true ]; then From 1433f23effeafbe422d4cd9f540600cd52e8da2a Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 13 May 2024 12:48:18 +0300 Subject: [PATCH 5/7] Add option --celldb-cache-size (#988) Co-authored-by: SpyCheese --- tddb/td/db/RocksDb.cpp | 30 +++++++++++++-------------- tddb/td/db/RocksDb.h | 8 ++++++- validator-engine/validator-engine.cpp | 13 ++++++++++++ validator-engine/validator-engine.hpp | 4 ++++ validator/db/archive-manager.cpp | 5 ++++- validator/db/archive-slice.cpp | 4 +++- validator/db/celldb.cpp | 8 ++++++- validator/validator-options.hpp | 7 +++++++ validator/validator.h | 2 ++ 9 files changed, 62 insertions(+), 19 deletions(-) diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index a84a804b..d15b8e19 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -59,38 +59,38 @@ RocksDb RocksDb::clone() const { return RocksDb{db_, statistics_}; } -Result RocksDb::open(std::string path, std::shared_ptr statistics) { +Result RocksDb::open(std::string path, RocksDbOptions options) { rocksdb::OptimisticTransactionDB *db; { - rocksdb::Options options; + rocksdb::Options db_options; - static auto cache = rocksdb::NewLRUCache(1 << 30); + static auto cache = rocksdb::NewLRUCache(options.block_cache_size); rocksdb::BlockBasedTableOptions table_options; table_options.block_cache = cache; - options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); - options.manual_wal_flush = true; - options.create_if_missing = true; - options.max_background_compactions = 4; - options.max_background_flushes = 2; - options.bytes_per_sync = 1 << 20; - options.writable_file_max_buffer_size = 2 << 14; - options.statistics = statistics; + db_options.manual_wal_flush = true; + db_options.create_if_missing = true; + db_options.max_background_compactions = 4; + db_options.max_background_flushes = 2; + db_options.bytes_per_sync = 1 << 20; + db_options.writable_file_max_buffer_size = 2 << 14; + db_options.statistics = options.statistics; rocksdb::OptimisticTransactionDBOptions occ_options; occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; - rocksdb::ColumnFamilyOptions cf_options(options); + rocksdb::ColumnFamilyOptions cf_options(db_options); std::vector column_families; column_families.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, cf_options)); std::vector handles; - TRY_STATUS(from_rocksdb( - rocksdb::OptimisticTransactionDB::Open(options, occ_options, std::move(path), column_families, &handles, &db))); + TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(db_options, occ_options, std::move(path), + column_families, &handles, &db))); CHECK(handles.size() == 1); // i can delete the handle since DBImpl is always holding a reference to // default column family delete handles[0]; } - return RocksDb(std::shared_ptr(db), std::move(statistics)); + return RocksDb(std::shared_ptr(db), std::move(options.statistics)); } std::shared_ptr RocksDb::create_statistics() { diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 1afba4cc..44ef9de6 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -34,11 +34,17 @@ class Statistics; } // namespace rocksdb namespace td { + +struct RocksDbOptions { + std::shared_ptr statistics = nullptr; + uint64 block_cache_size = 1 << 30; +}; + class RocksDb : public KeyValue { public: static Status destroy(Slice path); RocksDb clone() const; - static Result open(std::string path, std::shared_ptr statistics = nullptr); + static Result open(std::string path, RocksDbOptions options = {}); Result get(Slice key, std::string &value) override; Status set(Slice key, Slice value) override; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 3c98a4ac..15408510 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1369,6 +1369,9 @@ td::Status ValidatorEngine::load_global_config() { validator_options_.write().set_archive_preload_period(archive_preload_period_); validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); validator_options_.write().set_nonfinal_ls_queries_enabled(nonfinal_ls_queries_enabled_); + if (celldb_cache_size_) { + validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value()); + } std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -3967,6 +3970,16 @@ int main(int argc, char *argv[]) { p.add_option('\0', "nonfinal-ls", "enable special LS queries to non-finalized blocks", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); }); }); + p.add_checked_option( + '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)", + [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + if (v == 0) { + return td::Status::Error("celldb-cache-size should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 0a99dfcc..39685f9d 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -209,6 +209,7 @@ class ValidatorEngine : public td::actor::Actor { double archive_preload_period_ = 0.0; bool disable_rocksdb_stats_ = false; bool nonfinal_ls_queries_enabled_ = false; + td::optional celldb_cache_size_; bool read_config_ = false; bool started_keyring_ = false; bool started_ = false; @@ -281,6 +282,9 @@ class ValidatorEngine : public td::actor::Actor { void set_nonfinal_ls_queries_enabled() { nonfinal_ls_queries_enabled_ = true; } + void set_celldb_cache_size(td::uint64 value) { + celldb_cache_size_ = value; + } void start_up() override; ValidatorEngine() { } diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index c806f4e5..b87d04f7 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -832,7 +832,10 @@ void ArchiveManager::start_up() { if (!opts_->get_disable_rocksdb_stats()) { statistics_.init(); } - index_ = std::make_shared(td::RocksDb::open(db_root_ + "/files/globalindex", statistics_.rocksdb_statistics).move_as_ok()); + td::RocksDbOptions db_options; + db_options.statistics = statistics_.rocksdb_statistics; + index_ = std::make_shared( + td::RocksDb::open(db_root_ + "/files/globalindex", std::move(db_options)).move_as_ok()); std::string value; auto v = index_->get(create_serialize_tl_object().as_slice(), value); v.ensure(); diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index daee546b..d392431a 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -554,7 +554,9 @@ void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise(td::RocksDb::open(db_path_, statistics_.rocksdb_statistics).move_as_ok()); + td::RocksDbOptions db_options; + db_options.statistics = statistics_.rocksdb_statistics; + kv_ = std::make_unique(td::RocksDb::open(db_path_, std::move(db_options)).move_as_ok()); std::string value; auto R2 = kv_->get("status", value); R2.ensure(); diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 294515a3..007bedc8 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -88,7 +88,13 @@ void CellDbIn::start_up() { statistics_ = td::RocksDb::create_statistics(); statistics_flush_at_ = td::Timestamp::in(60.0); } - cell_db_ = std::make_shared(td::RocksDb::open(path_, statistics_).move_as_ok()); + td::RocksDbOptions db_options; + db_options.statistics = statistics_; + if (opts_->get_celldb_cache_size()) { + db_options.block_cache_size = opts_->get_celldb_cache_size().value(); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(db_options.block_cache_size); + } + cell_db_ = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); boc_ = vm::DynamicBagOfCellsDb::create(); diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 1b7c5b09..2dd85b56 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -129,6 +129,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool nonfinal_ls_queries_enabled() const override { return nonfinal_ls_queries_enabled_; } + td::optional get_celldb_cache_size() const override { + return celldb_cache_size_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -197,6 +200,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_nonfinal_ls_queries_enabled(bool value) override { nonfinal_ls_queries_enabled_ = value; } + void set_celldb_cache_size(td::uint64 value) override { + celldb_cache_size_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -244,6 +250,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double archive_preload_period_ = 0.0; bool disable_rocksdb_stats_; bool nonfinal_ls_queries_enabled_ = false; + td::optional celldb_cache_size_; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 17e55c70..92332e55 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -86,6 +86,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual double get_archive_preload_period() const = 0; virtual bool get_disable_rocksdb_stats() const = 0; virtual bool nonfinal_ls_queries_enabled() const = 0; + virtual td::optional get_celldb_cache_size() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -110,6 +111,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_archive_preload_period(double value) = 0; virtual void set_disable_rocksdb_stats(bool value) = 0; virtual void set_nonfinal_ls_queries_enabled(bool value) = 0; + virtual void set_celldb_cache_size(td::uint64 value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, From c7fd75ce56dbdb6d8aece97415275c09efa9318b Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 13 May 2024 14:54:57 +0300 Subject: [PATCH 6/7] Fix creating rocksdb cache (#989) Co-authored-by: SpyCheese --- tddb/td/db/RocksDb.cpp | 8 ++++++-- tddb/td/db/RocksDb.h | 3 ++- validator-engine/validator-engine.cpp | 3 ++- validator/db/celldb.cpp | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index d15b8e19..91c5ca66 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -64,10 +64,14 @@ Result RocksDb::open(std::string path, RocksDbOptions options) { { rocksdb::Options db_options; - static auto cache = rocksdb::NewLRUCache(options.block_cache_size); + static auto cache = rocksdb::NewLRUCache(1 << 30); rocksdb::BlockBasedTableOptions table_options; - table_options.block_cache = cache; + if (options.block_cache_size) { + table_options.block_cache = rocksdb::NewLRUCache(options.block_cache_size.value()); + } else { + table_options.block_cache = cache; + } db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); db_options.manual_wal_flush = true; diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 44ef9de6..babfadb7 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -24,6 +24,7 @@ #include "td/db/KeyValue.h" #include "td/utils/Status.h" +#include "td/utils/optional.h" namespace rocksdb { class OptimisticTransactionDB; @@ -37,7 +38,7 @@ namespace td { struct RocksDbOptions { std::shared_ptr statistics = nullptr; - uint64 block_cache_size = 1 << 30; + optional block_cache_size; // Default - one 1G cache for all RocksDb }; class RocksDb : public KeyValue { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 15408510..f52fe8cf 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3971,7 +3971,8 @@ int main(int argc, char *argv[]) { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); }); }); p.add_checked_option( - '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)", + '\0', "celldb-cache-size", + "block cache size for RocksDb in CellDb, in bytes (default: 1G cache shared by archive DB)", [&](td::Slice s) -> td::Status { TRY_RESULT(v, td::to_integer_safe(s)); if (v == 0) { diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 007bedc8..bc315b59 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -92,7 +92,7 @@ void CellDbIn::start_up() { db_options.statistics = statistics_; if (opts_->get_celldb_cache_size()) { db_options.block_cache_size = opts_->get_celldb_cache_size().value(); - LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(db_options.block_cache_size); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(db_options.block_cache_size.value()); } cell_db_ = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); From 816dd9cf2d7348f7646ce2befb366ac6165557ce Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 13 May 2024 14:55:32 +0300 Subject: [PATCH 7/7] Add option --catchain-max-block-delay (#990) Co-authored-by: SpyCheese --- validator-engine/validator-engine.cpp | 13 +++++++++++++ validator-engine/validator-engine.hpp | 4 ++++ validator-session/validator-session.cpp | 4 ++-- validator-session/validator-session.h | 1 + validator-session/validator-session.hpp | 5 +++++ validator/manager.cpp | 2 +- validator/validator-group.cpp | 4 ++++ validator/validator-group.hpp | 6 ++++-- validator/validator-options.hpp | 7 +++++++ validator/validator.h | 2 ++ 10 files changed, 43 insertions(+), 5 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index f52fe8cf..0c423f7a 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1372,6 +1372,9 @@ td::Status ValidatorEngine::load_global_config() { if (celldb_cache_size_) { validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value()); } + if (catchain_max_block_delay_) { + validator_options_.write().set_catchain_max_block_delay(catchain_max_block_delay_.value()); + } std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -3981,6 +3984,16 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); }); return td::Status::OK(); }); + p.add_checked_option( + '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v < 0) { + return td::Status::Error("catchain-max-block-delay should be non-negative"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 39685f9d..265bb9e4 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -210,6 +210,7 @@ class ValidatorEngine : public td::actor::Actor { bool disable_rocksdb_stats_ = false; bool nonfinal_ls_queries_enabled_ = false; td::optional celldb_cache_size_; + td::optional catchain_max_block_delay_; bool read_config_ = false; bool started_keyring_ = false; bool started_ = false; @@ -285,6 +286,9 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_cache_size(td::uint64 value) { celldb_cache_size_ = value; } + void set_catchain_max_block_delay(double value) { + catchain_max_block_delay_ = value; + } void start_up() override; ValidatorEngine() { } diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index d2fe6458..46dd4440 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -807,8 +807,8 @@ void ValidatorSessionImpl::request_new_block(bool now) { } else { double lambda = 10.0 / description().get_total_nodes(); double x = -1 / lambda * log(td::Random::fast(1, 999) * 0.001); - if (x > 0.5) { - x = 0.5; + if (x > catchain_max_block_delay_) { // default = 0.5 + x = catchain_max_block_delay_; } td::actor::send_closure(catchain_, &catchain::CatChain::need_new_block, td::Timestamp::in(x)); } diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 2dd6625c..0870f671 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -108,6 +108,7 @@ class ValidatorSession : public td::actor::Actor { virtual void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) = 0; + virtual void set_catchain_max_block_delay(double value) = 0; static td::actor::ActorOwn create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 005292c0..b6a9ab0c 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -90,6 +90,8 @@ class ValidatorSessionImpl : public ValidatorSession { td::actor::ActorOwn catchain_; std::unique_ptr description_; + double catchain_max_block_delay_ = 0.5; + void on_new_round(td::uint32 round); void on_catchain_started(); void check_vote_for_slot(td::uint32 att); @@ -188,6 +190,9 @@ class ValidatorSessionImpl : public ValidatorSession { void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) override; + void set_catchain_max_block_delay(double value) override { + catchain_max_block_delay_ = value; + } void process_blocks(std::vector blocks); void finished_processing(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 82cc10c2..3d544003 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2135,7 +2135,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto G = td::actor::create_actor( "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno())); + opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); return G; } } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 6a71d306..68e2b07e 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -348,6 +348,10 @@ void ValidatorGroup::create_session() { << ".", allow_unsafe_self_blocks_resync_); } + if (opts_->get_catchain_max_block_delay()) { + td::actor::send_closure(session_, &validatorsession::ValidatorSession::set_catchain_max_block_delay, + opts_->get_catchain_max_block_delay().value()); + } if (started_) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::start); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index cb3f7a63..f9940264 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -69,7 +69,7 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync) + bool allow_unsafe_self_blocks_resync, td::Ref opts) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -82,7 +82,8 @@ class ValidatorGroup : public td::actor::Actor { , db_root_(std::move(db_root)) , manager_(validator_manager) , init_(create_session) - , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { + , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) + , opts_(std::move(opts)) { } private: @@ -123,6 +124,7 @@ class ValidatorGroup : public td::actor::Actor { bool init_ = false; bool started_ = false; bool allow_unsafe_self_blocks_resync_; + td::Ref opts_; td::uint32 last_known_round_id_ = 0; struct CachedCollatedBlock { diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 2dd85b56..34f7fdf9 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -132,6 +132,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::optional get_celldb_cache_size() const override { return celldb_cache_size_; } + td::optional get_catchain_max_block_delay() const override { + return catchain_max_block_delay_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -203,6 +206,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_celldb_cache_size(td::uint64 value) override { celldb_cache_size_ = value; } + void set_catchain_max_block_delay(double value) override { + catchain_max_block_delay_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -251,6 +257,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool disable_rocksdb_stats_; bool nonfinal_ls_queries_enabled_ = false; td::optional celldb_cache_size_; + td::optional catchain_max_block_delay_; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 92332e55..81dfe7a3 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -87,6 +87,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual bool get_disable_rocksdb_stats() const = 0; virtual bool nonfinal_ls_queries_enabled() const = 0; virtual td::optional get_celldb_cache_size() const = 0; + virtual td::optional get_catchain_max_block_delay() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -112,6 +113,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_disable_rocksdb_stats(bool value) = 0; virtual void set_nonfinal_ls_queries_enabled(bool value) = 0; virtual void set_celldb_cache_size(td::uint64 value) = 0; + virtual void set_catchain_max_block_delay(double value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id,