From ce6c29941ea1e29a10c266ef615fb3302213642d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 13 Feb 2025 14:25:04 +0300 Subject: [PATCH 01/10] Get rid of std::cerr logs in collator/validator --- crypto/block/block.cpp | 7 +- crypto/block/mc-config.cpp | 14 +- crypto/block/output-queue-merger.cpp | 1 - crypto/block/transaction.cpp | 95 ++++++---- crypto/tl/tlblib.cpp | 7 + crypto/tl/tlblib.hpp | 7 + crypto/vm/cells/CellSlice.cpp | 7 + crypto/vm/cells/CellSlice.h | 1 + tdutils/td/utils/logging.h | 4 +- validator/impl/accept-block.cpp | 20 +- validator/impl/collator.cpp | 265 ++++++++++++++++----------- validator/impl/signature-set.cpp | 3 - validator/impl/top-shard-descr.cpp | 8 +- validator/impl/validate-query.cpp | 71 ++++--- 14 files changed, 310 insertions(+), 200 deletions(-) diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index 302a2aa4..452d78a2 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -360,7 +360,6 @@ MsgProcessedUptoCollection::MsgProcessedUptoCollection(ton::ShardIdFull _owner, z.shard = key.get_uint(64); z.mc_seqno = (unsigned)((key + 64).get_uint(32)); z.last_inmsg_lt = value.write().fetch_ulong(64); - // std::cerr << "ProcessedUpto shard " << std::hex << z.shard << std::dec << std::endl; return value.write().fetch_bits_to(z.last_inmsg_hash) && z.shard && ton::shard_contains(owner.shard, z.shard); }); } @@ -862,8 +861,10 @@ td::Status ShardState::unpack_out_msg_queue_info(Ref out_msg_queue_inf out_msg_queue_ = std::make_unique(std::move(qinfo.out_queue), 352, block::tlb::aug_OutMsgQueue); if (verbosity >= 3 * 1) { - LOG(DEBUG) << "unpacking ProcessedUpto of our previous block " << id_.to_str(); - block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info); + FLOG(DEBUG) { + sb << "unpacking ProcessedUpto of our previous block " << id_.to_str(); + block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info); + }; } if (!block::gen::t_ProcessedInfo.validate_csr(1024, qinfo.proc_info)) { return td::Status::Error( diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 14881913..48a2d613 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -163,8 +163,11 @@ td::Status ConfigInfo::unpack() { } gen::McStateExtra::Record extra_info; if (!tlb::unpack_cell(state_extra_root_, extra_info)) { - vm::load_cell_slice(state_extra_root_).print_rec(std::cerr); - block::gen::t_McStateExtra.print_ref(std::cerr, state_extra_root_); + FLOG(WARNING) { + sb << "state extra information is invalid: "; + vm::load_cell_slice(state_extra_root_).print_rec(sb); + block::gen::t_McStateExtra.print_ref(sb, state_extra_root_); + }; return td::Status::Error("state extra information is invalid"); } gen::ValidatorInfo::Record validator_info; @@ -1067,7 +1070,6 @@ Ref ShardConfig::get_shard_hash(ton::ShardIdFull id, bool exact) co ton::ShardIdFull true_id; vm::CellSlice cs; if (get_shard_hash_raw(cs, id, true_id, exact)) { - // block::gen::t_ShardDescr.print(std::cerr, vm::CellSlice{cs}); return McShardHash::unpack(cs, true_id); } else { return {}; @@ -1637,8 +1639,10 @@ bool ShardConfig::set_shard_info(ton::ShardIdFull shard, Ref value) { if (!gen::t_BinTree_ShardDescr.validate_ref(1024, value)) { LOG(ERROR) << "attempting to store an invalid (BinTree ShardDescr) at shard configuration position " << shard.to_str(); - gen::t_BinTree_ShardDescr.print_ref(std::cerr, value); - vm::load_cell_slice(value).print_rec(std::cerr); + FLOG(WARNING) { + gen::t_BinTree_ShardDescr.print_ref(sb, value); + vm::load_cell_slice(value).print_rec(sb); + }; return false; } auto root = shard_hashes_dict_->lookup_ref(td::BitArray<32>{shard.workchain}); diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index aa425f6b..7d258cfe 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -138,7 +138,6 @@ bool OutputQueueMerger::add_root(int src, Ref outmsg_root) { if (outmsg_root.is_null()) { return true; } - //block::gen::HashmapAug{352, block::gen::t_EnqueuedMsg, block::gen::t_uint64}.print_ref(std::cerr, outmsg_root); auto kv = std::make_unique(src, std::move(outmsg_root)); if (kv->replace_by_prefix(common_pfx.cbits(), common_pfx_len)) { heap.push_back(std::move(kv)); diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index ba50c581..46c696f5 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -446,8 +446,10 @@ bool Account::unpack(Ref shard_account, ton::UnixTime now, bool s return false; } if (verbosity > 2) { - shard_account->print_rec(std::cerr, 2); - block::gen::t_ShardAccount.print(std::cerr, *shard_account); + FLOG(INFO) { + shard_account->print_rec(sb, 2); + block::gen::t_ShardAccount.print(sb, shard_account); + }; } block::gen::ShardAccount::Record acc_info; if (!(block::tlb::t_ShardAccount.validate_csr(shard_account) && tlb::unpack_exact(shard_account.write(), acc_info))) { @@ -737,9 +739,11 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* return false; } if (verbosity > 2) { - fprintf(stderr, "unpacking inbound message for a new transaction: "); - block::gen::t_Message_Any.print_ref(std::cerr, in_msg); - load_cell_slice(in_msg).print_rec(std::cerr); + FLOG(INFO) { + sb << "unpacking inbound message for a new transaction: "; + block::gen::t_Message_Any.print_ref(sb, in_msg); + load_cell_slice(in_msg).print_rec(sb); + }; } auto cs = vm::load_cell_slice(in_msg); int tag = block::gen::t_CommonMsgInfo.get_tag(cs); @@ -1550,11 +1554,13 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom cp.actions = impl.get_c5(); int out_act_num = output_actions_count(cp.actions); if (verbosity > 2) { - std::cerr << "new smart contract data: "; - bool can_be_special = true; - load_cell_slice_special(cp.new_data, can_be_special).print_rec(std::cerr); - std::cerr << "output actions: "; - block::gen::OutList{out_act_num}.print_ref(std::cerr, cp.actions); + FLOG(INFO) { + sb << "new smart contract data: "; + bool can_be_special = true; + load_cell_slice_special(cp.new_data, can_be_special).print_rec(sb); + sb << "output actions: "; + block::gen::OutList{out_act_num}.print_ref(sb, cp.actions); + }; } } cp.mode = 0; @@ -1619,7 +1625,6 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { if (in_msg_state.not_null()) { LOG(DEBUG) << "HASH(in_msg_state) = " << in_msg_state->get_hash().bits().to_hex(256) << ", account_state_hash = " << account.state_hash.to_hex(); - // vm::load_cell_slice(in_msg_state).print_rec(std::cerr); } else { LOG(DEBUG) << "in_msg_state is null"; } @@ -1775,11 +1780,13 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { cp.actions = vm.get_committed_state().c5; // c5 -> action list int out_act_num = output_actions_count(cp.actions); if (verbosity > 2) { - std::cerr << "new smart contract data: "; - bool can_be_special = true; - load_cell_slice_special(cp.new_data, can_be_special).print_rec(std::cerr); - std::cerr << "output actions: "; - block::gen::OutList{out_act_num}.print_ref(std::cerr, cp.actions); + FLOG(INFO) { + sb << "new smart contract data: "; + bool can_be_special = true; + load_cell_slice_special(cp.new_data, can_be_special).print_rec(sb); + sb << "output actions: "; + block::gen::OutList{out_act_num}.print_ref(sb, cp.actions); + }; } } cp.mode = 0; @@ -2725,14 +2732,18 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, } if (!block::gen::t_Message_Any.validate_ref(new_msg)) { LOG(ERROR) << "generated outbound message is not a valid (Message Any) according to automated check"; - block::gen::t_Message_Any.print_ref(std::cerr, new_msg); - vm::load_cell_slice(new_msg).print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_Message_Any.print_ref(sb, new_msg); + vm::load_cell_slice(new_msg).print_rec(sb); + }; collect_fine(); return -1; } if (verbosity > 2) { - std::cerr << "converted outbound message: "; - block::gen::t_Message_Any.print_ref(std::cerr, new_msg); + FLOG(INFO) { + sb << "converted outbound message: "; + block::gen::t_Message_Any.print_ref(sb, new_msg); + }; } ap.msgs_created++; @@ -3045,8 +3056,10 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { } CHECK(cb.finalize_to(bp.out_msg)); if (verbosity > 2) { - LOG(INFO) << "generated bounced message: "; - block::gen::t_Message_Any.print_ref(std::cerr, bp.out_msg); + FLOG(INFO) { + sb << "generated bounced message: "; + block::gen::t_Message_Any.print_ref(sb, bp.out_msg); + }; } out_msgs.push_back(bp.out_msg); bp.ok = true; @@ -3167,11 +3180,13 @@ bool Transaction::compute_state() { auto frozen_state = cb2.finalize(); frozen_hash = frozen_state->get_hash().bits(); if (verbosity >= 3 * 1) { // !!!DEBUG!!! - std::cerr << "freezing state of smart contract: "; - block::gen::t_StateInit.print_ref(std::cerr, frozen_state); - CHECK(block::gen::t_StateInit.validate_ref(frozen_state)); - CHECK(block::tlb::t_StateInit.validate_ref(frozen_state)); - std::cerr << "with hash " << frozen_hash.to_hex() << std::endl; + FLOG(INFO) { + sb << "freezing state of smart contract: "; + block::gen::t_StateInit.print_ref(sb, frozen_state); + CHECK(block::gen::t_StateInit.validate_ref(frozen_state)); + CHECK(block::tlb::t_StateInit.validate_ref(frozen_state)); + sb << "with hash " << frozen_hash.to_hex(); + }; } } new_code.clear(); @@ -3229,8 +3244,10 @@ bool Transaction::compute_state() { CHECK(cb.append_data_cell_bool(std::move(storage))); new_total_state = cb.finalize(); if (verbosity > 2) { - std::cerr << "new account state: "; - block::gen::t_Account.print_ref(std::cerr, new_total_state); + FLOG(INFO) { + sb << "new account state: "; + block::gen::t_Account.print_ref(sb, new_total_state); + }; } CHECK(block::tlb::t_Account.validate_ref(new_total_state)); return true; @@ -3322,22 +3339,28 @@ bool Transaction::serialize() { return false; } if (verbosity >= 3 * 1) { - std::cerr << "new transaction: "; - block::gen::t_Transaction.print_ref(std::cerr, root); - vm::load_cell_slice(root).print_rec(std::cerr); + FLOG(INFO) { + sb << "new transaction: "; + block::gen::t_Transaction.print_ref(sb, root); + vm::load_cell_slice(root).print_rec(sb); + }; } if (!block::gen::t_Transaction.validate_ref(4096, root)) { LOG(ERROR) << "newly-generated transaction failed to pass automated validation:"; - vm::load_cell_slice(root).print_rec(std::cerr); - block::gen::t_Transaction.print_ref(std::cerr, root); + FLOG(INFO) { + vm::load_cell_slice(root).print_rec(sb); + block::gen::t_Transaction.print_ref(sb, root); + }; root.clear(); return false; } if (!block::tlb::t_Transaction.validate_ref(4096, root)) { LOG(ERROR) << "newly-generated transaction failed to pass hand-written validation:"; - vm::load_cell_slice(root).print_rec(std::cerr); - block::gen::t_Transaction.print_ref(std::cerr, root); + FLOG(INFO) { + vm::load_cell_slice(root).print_rec(sb); + block::gen::t_Transaction.print_ref(sb, root); + }; root.clear(); return false; } diff --git a/crypto/tl/tlblib.cpp b/crypto/tl/tlblib.cpp index 05ea8e1c..de5a483c 100644 --- a/crypto/tl/tlblib.cpp +++ b/crypto/tl/tlblib.cpp @@ -196,6 +196,13 @@ bool TLB::print_ref(std::ostream& os, Ref cell_ref, int indent, int re return pp.fail_unless(print_ref(pp, std::move(cell_ref))); } +bool TLB::print_ref(td::StringBuilder& sb, Ref cell_ref, int indent, int rec_limit) const { + std::ostringstream ss; + auto result = print_ref(ss, std::move(cell_ref), indent, rec_limit); + sb << ss.str(); + return result; +} + std::string TLB::as_string_skip(vm::CellSlice& cs, int indent) const { std::ostringstream os; print_skip(os, cs, indent); diff --git a/crypto/tl/tlblib.hpp b/crypto/tl/tlblib.hpp index a6350ece..c10049a9 100644 --- a/crypto/tl/tlblib.hpp +++ b/crypto/tl/tlblib.hpp @@ -246,7 +246,14 @@ class TLB { bool print(std::ostream& os, Ref cs_ref, int indent = 0, int rec_limit = 0) const { return print(os, *cs_ref, indent, rec_limit); } + bool print(td::StringBuilder& sb, Ref cs_ref, int indent = 0, int rec_limit = 0) const { + std::ostringstream ss; + auto result = print(ss, *cs_ref, indent, rec_limit); + sb << ss.str(); + return result; + } bool print_ref(std::ostream& os, Ref cell_ref, int indent = 0, int rec_limit = 0) const; + bool print_ref(td::StringBuilder& sb, Ref cell_ref, int indent = 0, int rec_limit = 0) const; bool print_ref(int rec_limit, std::ostream& os, Ref cell_ref, int indent = 0) const { return print_ref(os, std::move(cell_ref), indent, rec_limit); } diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 4d8c3c5a..9cd3e931 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -1026,6 +1026,13 @@ bool CellSlice::print_rec(std::ostream& os, int indent) const { return print_rec(os, &limit, indent); } +bool CellSlice::print_rec(td::StringBuilder& sb, int indent) const { + std::ostringstream ss; + auto result = print_rec(ss, indent); + sb << ss.str(); + return result; +} + bool CellSlice::print_rec(int limit, std::ostream& os, int indent) const { return print_rec(os, &limit, indent); } diff --git a/crypto/vm/cells/CellSlice.h b/crypto/vm/cells/CellSlice.h index 33fad741..ecce30f5 100644 --- a/crypto/vm/cells/CellSlice.h +++ b/crypto/vm/cells/CellSlice.h @@ -257,6 +257,7 @@ class CellSlice : public td::CntObject { void dump(std::ostream& os, int level = 0, bool endl = true) const; void dump_hex(std::ostream& os, int mode = 0, bool endl = false) const; bool print_rec(std::ostream& os, int indent = 0) const; + bool print_rec(td::StringBuilder& sb, int indent = 0) const; bool print_rec(std::ostream& os, int* limit, int indent = 0) const; bool print_rec(int limit, std::ostream& os, int indent = 0) const; void error() const { diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index dbf4c64b..bb28f6df 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -264,8 +264,8 @@ class Logger { sb_ << other; return *this; } - LambdaPrintHelper operator<<(const LambdaPrint &) { - return LambdaPrintHelper{*this}; + LambdaPrintHelper operator<<(const LambdaPrint &) { + return LambdaPrintHelper{sb_}; } MutableCSlice as_cslice() { diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index a9dd7fe2..de48626d 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -308,8 +308,11 @@ bool AcceptBlockQuery::create_new_proof() { } // 10. check resulting object if (!block::gen::t_BlockProof.validate_ref(bs_cell)) { - block::gen::t_BlockProof.print_ref(std::cerr, bs_cell); - vm::load_cell_slice(bs_cell).print_rec(std::cerr); + FLOG(WARNING) { + sb << "BlockProof object just created failed to pass automated consistency checks: "; + block::gen::t_BlockProof.print_ref(sb, bs_cell); + vm::load_cell_slice(bs_cell).print_rec(sb); + }; return fatal_error("BlockProof object just created failed to pass automated consistency checks"); } // 11. create a proof object from this cell @@ -851,15 +854,12 @@ bool AcceptBlockQuery::create_top_shard_block_description() { && (root.is_null() || cb.store_ref_bool(std::move(root))) && cb.finalize_to(td_cell))) { return fatal_error("cannot serialize ShardTopBlockDescription for the newly-accepted block "s + id_.to_str()); } - if (false) { - // debug output - std::cerr << "new ShardTopBlockDescription: "; - block::gen::t_TopBlockDescr.print_ref(std::cerr, td_cell); - vm::load_cell_slice(td_cell).print_rec(std::cerr); - } if (!block::gen::t_TopBlockDescr.validate_ref(td_cell)) { - block::gen::t_TopBlockDescr.print_ref(std::cerr, td_cell); - vm::load_cell_slice(td_cell).print_rec(std::cerr); + FLOG(WARNING) { + sb << "just created ShardTopBlockDescription is invalid: "; + block::gen::t_TopBlockDescr.print_ref(sb, td_cell); + vm::load_cell_slice(td_cell).print_rec(sb); + }; return fatal_error("just created ShardTopBlockDescription for "s + id_.to_str() + " is invalid"); } auto res = vm::std_boc_serialize(td_cell, 0); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index e171eaa1..d3378cd8 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -53,16 +53,6 @@ static constexpr int HIGH_PRIORITY_EXTERNAL = 10; // don't skip high priority e static constexpr int MAX_ATTEMPTS = 5; -#define DBG(__n) dbg(__n)&& -#define DSTART int __dcnt = 0; -#define DEB DBG(++__dcnt) - -static inline bool dbg(int c) TD_UNUSED; -static inline bool dbg(int c) { - std::cerr << '[' << (char)('0' + c / 10) << (char)('0' + c % 10) << ']'; - return true; -} - /** * Constructs a Collator object. * @@ -761,8 +751,6 @@ bool Collator::unpack_last_mc_state() { << " (upgrade validator software?)"; } // TODO: extract start_lt and end_lt from prev_mc_block as well - // std::cerr << " block::gen::ShardState::print_ref(mc_state_root) = "; - // block::gen::t_ShardState.print_ref(std::cerr, mc_state_root, 2); return true; } @@ -888,8 +876,10 @@ void Collator::got_neighbor_out_queue(int i, td::Result> res) // unpack ProcessedUpto LOG(DEBUG) << "unpacking ProcessedUpto of neighbor " << descr.blk_.to_str(); if (verbosity >= 2) { - block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info); - qinfo.proc_info->print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info); + qinfo.proc_info->print_rec(sb); + }; } descr.processed_upto = block::MsgProcessedUptoCollection::unpack(descr.shard(), qinfo.proc_info); if (!descr.processed_upto) { @@ -1756,9 +1746,11 @@ bool Collator::import_new_shard_top_blocks() { shard_conf_adjusted_ = true; } if (tb_act && verbosity >= 0) { // DEBUG - LOG(INFO) << "updated shard block configuration to "; - auto csr = shard_conf_->get_root_csr(); - block::gen::t_ShardHashes.print(std::cerr, csr.write()); + FLOG(INFO) { + sb << "updated shard block configuration to "; + auto csr = shard_conf_->get_root_csr(); + block::gen::t_ShardHashes.print(sb, csr); + }; } block::gen::ShardFeeCreated::Record fc; if (!(tlb::csr_unpack(fees_import_dict_->get_root_extra(), @@ -2279,10 +2271,12 @@ bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime deli bool Collator::out_msg_queue_cleanup() { LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors"; if (verbosity >= 2) { - auto rt = out_msg_queue_->get_root(); - std::cerr << "old out_msg_queue is "; - block::gen::t_OutMsgQueue.print(std::cerr, *rt); - rt->print_rec(std::cerr); + FLOG(INFO) { + auto rt = out_msg_queue_->get_root(); + sb << "old out_msg_queue is "; + block::gen::t_OutMsgQueue.print(sb, rt); + rt->print_rec(sb); + }; } if (after_merge_) { @@ -2422,10 +2416,12 @@ bool Collator::out_msg_queue_cleanup() { << out_msg_queue_size_; } if (verbosity >= 2) { - auto rt = out_msg_queue_->get_root(); - std::cerr << "new out_msg_queue is "; - block::gen::t_OutMsgQueue.print(std::cerr, *rt); - rt->print_rec(std::cerr); + FLOG(INFO) { + auto rt = out_msg_queue_->get_root(); + sb << "new out_msg_queue is "; + block::gen::t_OutMsgQueue.print(sb, rt); + rt->print_rec(sb); + }; } return register_out_msg_queue_op(true); } @@ -2524,19 +2520,27 @@ bool Collator::combine_account_transactions() { auto cell = cb.finalize(); auto csr = vm::load_cell_slice_ref(cell); if (verbosity > 2) { - std::cerr << "new AccountBlock for " << z.first.to_hex() << ": "; - block::gen::t_AccountBlock.print_ref(std::cerr, cell); - csr->print_rec(std::cerr); + FLOG(INFO) { + sb << "new AccountBlock for " << z.first.to_hex() << ": "; + block::gen::t_AccountBlock.print_ref(sb, cell); + csr->print_rec(sb); + }; } if (!block::gen::t_AccountBlock.validate_ref(100000, cell)) { - block::gen::t_AccountBlock.print_ref(std::cerr, cell); - csr->print_rec(std::cerr); + FLOG(WARNING) { + sb << "AccountBlock failed to pass automatic validation tests: "; + block::gen::t_AccountBlock.print_ref(sb, cell); + csr->print_rec(sb); + }; return fatal_error(std::string{"new AccountBlock for "} + z.first.to_hex() + " failed to pass automatic validation tests"); } if (!block::tlb::t_AccountBlock.validate_ref(100000, cell)) { - block::gen::t_AccountBlock.print_ref(std::cerr, cell); - csr->print_rec(std::cerr); + FLOG(WARNING) { + sb << "AccountBlock failed to pass handwritten validation tests: "; + block::gen::t_AccountBlock.print_ref(sb, cell); + csr->print_rec(sb); + }; return fatal_error(std::string{"new AccountBlock for "} + z.first.to_hex() + " failed to pass handwritten validation tests"); } @@ -2561,8 +2565,10 @@ bool Collator::combine_account_transactions() { } else if (acc.status == block::Account::acc_nonexist) { // account deleted if (verbosity > 2) { - std::cerr << "deleting account " << acc.addr.to_hex() << " with empty new value "; - block::gen::t_Account.print_ref(std::cerr, acc.total_state); + FLOG(INFO) { + sb << "deleting account " << acc.addr.to_hex() << " with empty new value "; + block::gen::t_Account.print_ref(sb, acc.total_state); + }; } if (account_dict->lookup_delete(acc.addr).is_null()) { return fatal_error(std::string{"cannot delete account "} + acc.addr.to_hex() + " from ShardAccounts"); @@ -2570,8 +2576,10 @@ bool Collator::combine_account_transactions() { } else { // existing account modified if (verbosity > 4) { - std::cerr << "modifying account " << acc.addr.to_hex() << " to "; - block::gen::t_Account.print_ref(std::cerr, acc.total_state); + FLOG(INFO) { + sb << "modifying account " << acc.addr.to_hex() << " to "; + block::gen::t_Account.print_ref(sb, acc.total_state); + }; } if (!(cb.store_ref_bool(acc.total_state) // account_descr$_ account:^Account && cb.store_bits_bool(acc.last_trans_hash_) // last_trans_hash:bits256 @@ -2594,9 +2602,11 @@ bool Collator::combine_account_transactions() { return fatal_error("cannot serialize ShardAccountBlocks"); } if (verbosity > 2) { - std::cerr << "new ShardAccountBlocks: "; - block::gen::t_ShardAccountBlocks.print_ref(std::cerr, shard_account_blocks_); - vm::load_cell_slice(shard_account_blocks_).print_rec(std::cerr); + FLOG(INFO) { + sb << "new ShardAccountBlocks: "; + block::gen::t_ShardAccountBlocks.print_ref(sb, shard_account_blocks_); + vm::load_cell_slice(shard_account_blocks_).print_rec(sb); + }; } if (!block::gen::t_ShardAccountBlocks.validate_ref(100000, shard_account_blocks_)) { return fatal_error("new ShardAccountBlocks failed to pass automatic validity tests"); @@ -2606,9 +2616,11 @@ bool Collator::combine_account_transactions() { } auto shard_accounts = account_dict->get_root(); if (verbosity > 2) { - std::cerr << "new ShardAccounts: "; - block::gen::t_ShardAccounts.print(std::cerr, *shard_accounts); - shard_accounts->print_rec(std::cerr); + FLOG(INFO) { + sb << "new ShardAccounts: "; + block::gen::t_ShardAccounts.print(sb, shard_accounts); + shard_accounts->print_rec(sb); + }; } if (verify >= 2) { LOG(INFO) << "verifying new ShardAccounts"; @@ -2659,7 +2671,9 @@ bool Collator::create_special_transaction(block::CurrencyCollection amount, Ref< addr.to_hex()); } if (verbosity >= 4) { - block::gen::t_Message_Any.print_ref(std::cerr, msg); + FLOG(INFO) { + block::gen::t_Message_Any.print_ref(sb, msg); + }; } CHECK(block::gen::t_Message_Any.validate_ref(msg)); CHECK(block::tlb::t_Message.validate_ref(msg)); @@ -3163,8 +3177,10 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R Ref msg_env; CHECK(block::tlb::pack_cell(msg_env, msg_env_rec)); if (verbosity > 2) { - std::cerr << "new (processed outbound) message envelope: "; - block::gen::t_MsgEnvelope.print_ref(std::cerr, msg_env); + FLOG(INFO) { + sb << "new (processed outbound) message envelope: "; + block::gen::t_MsgEnvelope.print_ref(sb, msg_env); + }; } // 3. create InMsg, referring to this MsgEnvelope and this Transaction vm::CellBuilder cb; @@ -3286,16 +3302,20 @@ bool Collator::enqueue_transit_message(Ref msg, Ref old_msg_ Ref out_msg = cb.finalize(); // 4.1. insert OutMsg into OutMsgDescr if (verbosity > 2) { - std::cerr << "OutMsg for a transit message: "; - block::gen::t_OutMsg.print_ref(std::cerr, out_msg); + FLOG(INFO) { + sb << "OutMsg for a transit message: "; + block::gen::t_OutMsg.print_ref(sb, out_msg); + }; } if (!insert_out_msg(out_msg)) { return fatal_error("cannot insert a new OutMsg into OutMsgDescr"); } // 4.2. insert InMsg into InMsgDescr if (verbosity > 2) { - std::cerr << "InMsg for a transit message: "; - block::gen::t_InMsg.print_ref(std::cerr, in_msg); + FLOG(INFO) { + sb << "InMsg for a transit message: "; + block::gen::t_InMsg.print_ref(sb, in_msg); + }; } if (!insert_in_msg(in_msg)) { return fatal_error("cannot insert a new InMsg into InMsgDescr"); @@ -3366,7 +3386,10 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 || (enqueued_lt = enq_msg->prefetch_ulong(64)) < /* 0 */ 1 * lt) { // DEBUG if (enq_msg.not_null()) { - block::gen::t_EnqueuedMsg.print(std::cerr, *enq_msg); + FLOG(WARNING) { + sb << "inbound internal message is not a valid EnqueuedMsg: "; + block::gen::t_EnqueuedMsg.print(sb, enq_msg); + }; } LOG(ERROR) << "inbound internal message is not a valid EnqueuedMsg (created lt " << lt << ", enqueued " << enqueued_lt << ")"; @@ -3590,14 +3613,18 @@ bool Collator::process_inbound_internal_messages() { LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; if (verbosity > 2) { - std::cerr << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } if (!process_inbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source))) { if (verbosity > 1) { - std::cerr << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() - << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() + << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } return fatal_error("error processing inbound internal message"); } @@ -3884,7 +3911,10 @@ bool Collator::process_deferred_message(Ref enq_msg, StdSmcAddres LogicalTime enqueued_lt = 0; if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 || (enqueued_lt = enq_msg->prefetch_ulong(64)) != lt) { if (enq_msg.not_null()) { - block::gen::t_EnqueuedMsg.print(std::cerr, *enq_msg); + FLOG(WARNING) { + sb << "internal message in DispatchQueue is not a valid EnqueuedMsg: "; + block::gen::t_EnqueuedMsg.print(sb, enq_msg); + }; } LOG(ERROR) << "internal message in DispatchQueue is not a valid EnqueuedMsg (created lt " << lt << ", enqueued " << enqueued_lt << ")"; @@ -3986,8 +4016,10 @@ bool Collator::process_deferred_message(Ref enq_msg, StdSmcAddres */ bool Collator::insert_in_msg(Ref in_msg) { if (verbosity > 2) { - std::cerr << "InMsg being inserted into InMsgDescr: "; - block::gen::t_InMsg.print_ref(std::cerr, in_msg); + FLOG(INFO) { + sb << "InMsg being inserted into InMsgDescr: "; + block::gen::t_InMsg.print_ref(sb, in_msg); + }; } auto cs = load_cell_slice(in_msg); if (!cs.size_refs()) { @@ -4028,8 +4060,10 @@ bool Collator::insert_in_msg(Ref in_msg) { */ bool Collator::insert_out_msg(Ref out_msg) { if (verbosity > 2) { - std::cerr << "OutMsg being inserted into OutMsgDescr: "; - block::gen::t_OutMsg.print_ref(std::cerr, out_msg); + FLOG(INFO) { + sb << "OutMsg being inserted into OutMsgDescr: "; + block::gen::t_OutMsg.print_ref(sb, out_msg); + }; } auto cs = load_cell_slice(out_msg); if (!cs.size_refs()) { @@ -4125,8 +4159,10 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema } // 4. insert OutMsg into OutMsgDescr if (verbosity > 2) { - std::cerr << "OutMsg for a newly-generated message: "; - block::gen::t_OutMsg.print_ref(std::cerr, out_msg); + FLOG(INFO) { + sb << "OutMsg for a newly-generated message: "; + block::gen::t_OutMsg.print_ref(sb, out_msg); + }; } if (!insert_out_msg(out_msg)) { return fatal_error("cannot insert a new OutMsg into OutMsgDescr"); @@ -4419,9 +4455,12 @@ bool Collator::create_mc_state_extra() { bool ignore_cfg_changes = false; Ref cfg0; if (!block::valid_config_data(cfg_smc_config, config_addr, true, true, old_mparams_)) { - block::gen::t_Hashmap_32_Ref_Cell.print_ref(std::cerr, cfg_smc_config); LOG(ERROR) << "configuration smart contract "s + config_addr.to_hex() + " contains an invalid configuration in its data, IGNORING CHANGES"; + FLOG(WARNING) { + sb << "ignored configuration: "; + block::gen::t_Hashmap_32_Ref_Cell.print_ref(sb, cfg_smc_config); + }; ignore_cfg_changes = true; } else { cfg0 = cfg_dict.lookup_ref(td::BitArray<32>{(long long)0}); @@ -4459,34 +4498,26 @@ bool Collator::create_mc_state_extra() { return fatal_error(wset_res.move_as_error()); } bool update_shard_cc = is_key_block_ || (now_ / ccvc.shard_cc_lifetime > prev_now_ / ccvc.shard_cc_lifetime); - // temp debug - if (verbosity >= 3 * 1) { - auto csr = shard_conf_->get_root_csr(); - LOG(INFO) << "new shard configuration before post-processing is"; - std::ostringstream os; - csr->print_rec(os); - block::gen::t_ShardHashes.print(os, csr.write()); - LOG(INFO) << os.str(); - } - // end (temp debug) if (!update_shard_config(wset_res.move_as_ok(), ccvc, update_shard_cc)) { auto csr = shard_conf_->get_root_csr(); if (csr.is_null()) { LOG(WARNING) << "new shard configuration is null (!)"; } else { LOG(WARNING) << "invalid new shard configuration is"; - std::ostringstream os; - csr->print_rec(os); - block::gen::t_ShardHashes.print(os, csr.write()); - LOG(WARNING) << os.str(); + FLOG(WARNING) { + csr->print_rec(sb); + block::gen::t_ShardHashes.print(sb, csr); + }; } return fatal_error("cannot post-process shard configuration"); } // 3. save new shard_hashes state_extra.shard_hashes = shard_conf_->get_root_csr(); - if (verbosity >= 3 * 0) { // DEBUG - std::cerr << "updated shard configuration to "; - block::gen::t_ShardHashes.print(std::cerr, *state_extra.shard_hashes); + if (verbosity >= 3) { + FLOG(INFO) { + sb << "updated shard configuration to "; + block::gen::t_ShardHashes.print(sb, state_extra.shard_hashes); + }; } if (!block::gen::t_ShardHashes.validate_upto(10000, *state_extra.shard_hashes)) { return fatal_error("new ShardHashes is invalid"); @@ -4587,13 +4618,18 @@ bool Collator::create_mc_state_extra() { if (verify >= 2) { LOG(INFO) << "verifying new BlockCreateStats"; if (!block::gen::t_BlockCreateStats.validate_csr(100000, cs)) { - cs->print_rec(std::cerr); - block::gen::t_BlockCreateStats.print(std::cerr, *cs); + FLOG(WARNING) { + sb << "BlockCreateStats in the new masterchain state failed to pass automated validity checks: "; + cs->print_rec(sb); + block::gen::t_BlockCreateStats.print(sb, cs); + }; return fatal_error("BlockCreateStats in the new masterchain state failed to pass automated validity checks"); } } if (verbosity >= 4 * 1) { - block::gen::t_BlockCreateStats.print(std::cerr, *cs); + FLOG(INFO) { + block::gen::t_BlockCreateStats.print(sb, cs); + }; } } else { state_extra.r1.block_create_stats.clear(); @@ -4628,7 +4664,6 @@ bool Collator::update_block_creator_count(td::ConstBitPtr key, unsigned shard_in if (!block::unpack_CreatorStats(std::move(cs), mc_cnt, shard_cnt)) { return fatal_error("cannot unpack CreatorStats for "s + key.to_hex(256) + " from previous masterchain state"); } - // std::cerr << mc_cnt.to_str() << " " << shard_cnt.to_str() << std::endl; if (mc_incr && !mc_cnt.increase_by(mc_incr, now_)) { return fatal_error(PSTRING() << "cannot increase masterchain block counter in CreatorStats for " << key.to_hex(256) << " by " << mc_incr << " (old value is " << mc_cnt.to_str() << ")"); @@ -4999,9 +5034,11 @@ bool Collator::update_public_libraries() { } } if (libraries_changed_ && verbosity >= 2) { - std::cerr << "New public libraries: "; - block::gen::t_HashmapE_256_LibDescr.print(std::cerr, shard_libraries_->get_root()); - shard_libraries_->get_root()->print_rec(std::cerr); + FLOG(INFO) { + sb << "New public libraries: "; + block::gen::t_HashmapE_256_LibDescr.print(sb, shard_libraries_->get_root()); + shard_libraries_->get_root()->print_rec(sb); + }; } return true; } @@ -5124,9 +5161,11 @@ bool Collator::create_shard_state() { } LOG(DEBUG) << "min_ref_mc_seqno is " << min_ref_mc_seqno_; if (verbosity > 2) { - std::cerr << "new ShardState: "; - block::gen::t_ShardState.print_ref(std::cerr, state_root); - vm::load_cell_slice(state_root).print_rec(std::cerr); + FLOG(INFO) { + sb << "new ShardState: "; + block::gen::t_ShardState.print_ref(sb, state_root); + vm::load_cell_slice(state_root).print_rec(sb); + }; } if (verify >= 2) { LOG(INFO) << "verifying new ShardState"; @@ -5139,9 +5178,11 @@ bool Collator::create_shard_state() { return fatal_error("cannot create Merkle update for ShardState"); } if (verbosity > 2) { - std::cerr << "Merkle Update for ShardState: "; - vm::CellSlice cs{vm::NoVm{}, state_update}; - cs.print_rec(std::cerr); + FLOG(INFO) { + sb << "Merkle Update for ShardState: "; + vm::CellSlice cs{vm::NoVm{}, state_update}; + cs.print_rec(sb); + }; } LOG(INFO) << "updating block profile statistics"; block_limit_status_->add_proof(state_root); @@ -5186,10 +5227,12 @@ bool Collator::update_processed_upto() { */ bool Collator::compute_out_msg_queue_info(Ref& out_msg_queue_info) { if (verbosity >= 2) { - auto rt = out_msg_queue_->get_root(); - std::cerr << "resulting out_msg_queue is "; - block::gen::t_OutMsgQueue.print(std::cerr, *rt); - rt->print_rec(std::cerr); + FLOG(INFO) { + auto rt = out_msg_queue_->get_root(); + sb << "resulting out_msg_queue is "; + block::gen::t_OutMsgQueue.print(sb, rt); + rt->print_rec(sb); + }; } vm::CellBuilder cb; // out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) = OutMsgQueueExtra; @@ -5239,8 +5282,10 @@ bool Collator::compute_total_balance() { } vm::CellSlice cs{*(in_msg_dict->get_root_extra())}; if (verbosity > 2) { - block::gen::t_ImportFees.print(std::cerr, vm::CellSlice{*(in_msg_dict->get_root_extra())}); - cs.print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_ImportFees.print(sb, in_msg_dict->get_root_extra()); + cs.print_rec(sb); + }; } auto new_import_fees = block::tlb::t_Grams.as_integer_skip(cs); if (new_import_fees.is_null()) { @@ -5468,9 +5513,11 @@ bool Collator::create_block() { return fatal_error("cannot create new Block"); } if (verbosity >= 3 * 1) { - std::cerr << "new Block: "; - block::gen::t_Block.print_ref(std::cerr, new_block); - vm::load_cell_slice(new_block).print_rec(std::cerr); + FLOG(INFO) { + sb << "new Block: "; + block::gen::t_Block.print_ref(sb, new_block); + vm::load_cell_slice(new_block).print_rec(sb); + }; } if (verify >= 1) { LOG(INFO) << "verifying new Block"; @@ -5508,9 +5555,11 @@ Ref Collator::collate_shard_block_descr_set() { return {}; } if (verbosity >= 4 * 1) { - std::cerr << "serialized TopBlockDescrSet for collated data is: "; - block::gen::t_TopBlockDescrSet.print_ref(std::cerr, cell); - vm::load_cell_slice(cell).print_rec(std::cerr); + FLOG(INFO) { + sb << "serialized TopBlockDescrSet for collated data is: "; + block::gen::t_TopBlockDescrSet.print_ref(sb, cell); + vm::load_cell_slice(cell).print_rec(sb); + }; } return cell; } @@ -5717,8 +5766,10 @@ td::Result Collator::register_external_message_cell(Ref ext_msg, return td::Status::Error("inbound external message has destination address not in this shard"); } if (verbosity > 2) { - std::cerr << "registered external message: "; - block::gen::t_Message_Any.print_ref(std::cerr, ext_msg); + FLOG(INFO) { + sb << "registered external message: "; + block::gen::t_Message_Any.print_ref(sb, ext_msg); + }; } ext_msg_map.emplace(hash, 1); ext_msg_list_.push_back({std::move(ext_msg), ext_hash, priority}); diff --git a/validator/impl/signature-set.cpp b/validator/impl/signature-set.cpp index c7298216..0078a115 100644 --- a/validator/impl/signature-set.cpp +++ b/validator/impl/signature-set.cpp @@ -42,9 +42,6 @@ td::BufferSlice BlockSignatureSetQ::serialize() const { } Ref root; CHECK(serialize_to(root)); - //std::cerr << "serializing BlockSignatureSet: "; - //vm::CellSlice{vm::NoVm{}, root}.print_rec(std::cerr); - //std::cerr << std::endl; auto res = vm::std_boc_serialize(std::move(root)); LOG_CHECK(res.is_ok()) << res.move_as_error(); return res.move_as_ok(); diff --git a/validator/impl/top-shard-descr.cpp b/validator/impl/top-shard-descr.cpp index 8ff8862d..9eadeef3 100644 --- a/validator/impl/top-shard-descr.cpp +++ b/validator/impl/top-shard-descr.cpp @@ -175,9 +175,11 @@ td::Status ShardTopBlockDescrQ::unpack() { block::gen::TopBlockDescr::Record rec; if (!(block::gen::t_TopBlockDescr.force_validate_ref(root_) && tlb::unpack_cell(root_, rec) && block::tlb::t_BlockIdExt.unpack(rec.proof_for.write(), block_id_))) { - std::cerr << "invalid ShardTopBlockDescr: "; - block::gen::t_TopBlockDescr.print_ref(std::cerr, root_); - vm::load_cell_slice(root_).print_rec(std::cerr); + FLOG(INFO) { + sb << "invalid ShardTopBlockDescr: "; + block::gen::t_TopBlockDescr.print_ref(sb, root_); + vm::load_cell_slice(root_).print_rec(sb); + }; return td::Status::Error(-666, "Shard top block description is not a valid TopBlockDescr TL-B object"); } LOG(DEBUG) << "unpacking a ShardTopBlockDescr for " << block_id_.to_str() << " with " << rec.len << " links"; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 9e4d406e..583b1d86 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1553,8 +1553,10 @@ void ValidateQuery::got_neighbor_out_queue(int i, td::Result> // unpack ProcessedUpto LOG(DEBUG) << "unpacking ProcessedUpto of neighbor " << descr.blk_.to_str(); if (verbosity >= 2) { - block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info); - qinfo.proc_info->print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info); + qinfo.proc_info->print_rec(sb); + }; } descr.processed_upto = block::MsgProcessedUptoCollection::unpack(descr.shard(), qinfo.proc_info); if (!descr.processed_upto) { @@ -2656,7 +2658,6 @@ bool ValidateQuery::unpack_precheck_value_flow(Ref value_flow_root) { " but the sum over all accounts present in the new state is " + cc.to_str()); } auto msg_extra = in_msg_dict_->get_root_extra(); - // block::gen::t_ImportFees.print(std::cerr, msg_extra); if (!(block::tlb::t_Grams.as_integer_skip_to(msg_extra.write(), import_fees_) && cc.unpack(std::move(msg_extra)))) { return reject_query("cannot unpack ImportFees from the augmentation of the InMsgDescr dictionary"); } @@ -2760,20 +2761,22 @@ bool ValidateQuery::precheck_one_account_update(td::ConstBitPtr acc_id, Reflookup(acc_id, 256); if (acc_blk_root.is_null()) { if (verbosity >= 3 * 0) { - std::cerr << "state of account " << workchain() << ":" << acc_id.to_hex(256) - << " in the old shardchain state:" << std::endl; - if (old_value.not_null()) { - block::gen::t_ShardAccount.print(std::cerr, *old_value); - } else { - std::cerr << "" << std::endl; - } - std::cerr << "state of account " << workchain() << ":" << acc_id.to_hex(256) - << " in the new shardchain state:" << std::endl; - if (new_value.not_null()) { - block::gen::t_ShardAccount.print(std::cerr, *new_value); - } else { - std::cerr << "" << std::endl; - } + FLOG(INFO) { + sb << "state of account " << workchain() << ":" << acc_id.to_hex(256) + << " in the old shardchain state:" << "\n"; + if (old_value.not_null()) { + block::gen::t_ShardAccount.print(sb, old_value); + } else { + sb << "" << "\n"; + } + sb << "state of account " << workchain() << ":" << acc_id.to_hex(256) + << " in the new shardchain state:" << "\n"; + if (new_value.not_null()) { + block::gen::t_ShardAccount.print(sb, new_value); + } else { + sb << "" << "\n"; + } + }; } return reject_query("the state of account "s + acc_id.to_hex(256) + " changed in the new state with respect to the old state, but the block contains no " @@ -2931,8 +2934,6 @@ bool ValidateQuery::precheck_one_account_block(td::ConstBitPtr acc_id, Refprint_rec(std::cerr); - // block::gen::t_AccountBlock.print(std::cerr, acc_blk_root); block::gen::AccountBlock::Record acc_blk; block::gen::HASH_UPDATE::Record hash_upd; if (!(tlb::csr_unpack(acc_blk_root, acc_blk) && @@ -3860,7 +3861,9 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) ton::LogicalTime trans_lt; CHECK(block::get_transaction_id(transaction, trans_addr, trans_lt)); if (dest_addr != trans_addr) { - block::gen::t_InMsg.print(std::cerr, *in_msg); + FLOG(INFO) { + block::gen::t_InMsg.print(sb, in_msg); + }; return reject_query(PSTRING() << "InMsg corresponding to inbound message with hash " << key.to_hex(256) << " and destination address " << dest_addr.to_hex() << " claims that the message is processed by transaction " << trans_lt @@ -4408,7 +4411,9 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms ton::LogicalTime trans_lt; CHECK(block::get_transaction_id(transaction, trans_addr, trans_lt)); if (src_addr != trans_addr) { - block::gen::t_OutMsg.print(std::cerr, *out_msg); + FLOG(INFO) { + block::gen::t_OutMsg.print(sb, out_msg); + }; return reject_query(PSTRING() << "OutMsg corresponding to outbound message with hash " << key.to_hex(256) << " and source address " << src_addr.to_hex() << " claims that the message was created by transaction " << trans_lt @@ -5022,15 +5027,19 @@ bool ValidateQuery::check_in_queue() { LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; if (verbosity > 3) { - std::cerr << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } bool unprocessed = false; if (!check_neighbor_outbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source), unprocessed)) { if (verbosity > 1) { - std::cerr << "invalid neighbor outbound message: lt=" << kv->lt << " from=" << kv->source - << " key=" << kv->key.to_hex() << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "invalid neighbor outbound message: lt=" << kv->lt << " from=" << kv->source + << " key=" << kv->key.to_hex() << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } return reject_query("error processing outbound internal message "s + kv->key.to_hex() + " of neighbor " + neighbors_.at(kv->source).blk_.to_str()); @@ -5636,10 +5645,12 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // now compare the re-created transaction with the one we have if (trans_root2->get_hash() != trans_root->get_hash()) { if (verbosity >= 3 * 0) { - std::cerr << "original transaction " << lt << " of " << addr.to_hex() << ": "; - block::gen::t_Transaction.print_ref(std::cerr, trans_root); - std::cerr << "re-created transaction " << lt << " of " << addr.to_hex() << ": "; - block::gen::t_Transaction.print_ref(std::cerr, trans_root2); + FLOG(INFO) { + sb << "original transaction " << lt << " of " << addr.to_hex() << ": "; + block::gen::t_Transaction.print_ref(sb, trans_root); + sb << "re-created transaction " << lt << " of " << addr.to_hex() << ": "; + block::gen::t_Transaction.print_ref(sb, trans_root2); + }; } return reject_query(PSTRING() << "the transaction " << lt << " of " << addr.to_hex() << " has hash " << trans_root->get_hash().to_hex() From 9d94e04d2031ab0200699d7aec7f4a97fd1d0563 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 17 Feb 2025 10:13:17 +0300 Subject: [PATCH 02/10] Add more stats to validator getstats 1) Liteserver queries count 2) Collated/validated blocks count, number of active sessions 3) Persistent state sizes 4) Initial sync progress --- tdutils/td/utils/Time.h | 4 + validator-engine/validator-engine.cpp | 10 ++- validator-engine/validator-engine.hpp | 4 + validator/db/archive-manager.cpp | 24 ++++++ validator/db/archive-manager.hpp | 2 + validator/db/celldb.cpp | 19 +++- validator/db/rootdb.cpp | 1 + validator/downloaders/download-state.cpp | 7 ++ validator/downloaders/download-state.hpp | 3 + validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 2 + validator/impl/liteserver.cpp | 14 +-- validator/impl/liteserver.hpp | 1 - validator/impl/validate-query.cpp | 12 +-- validator/impl/validate-query.hpp | 2 +- validator/interfaces/validator-manager.h | 6 +- validator/manager-init.cpp | 6 ++ validator/manager-init.hpp | 4 + validator/manager.cpp | 82 +++++++++++++++--- validator/manager.hpp | 26 +++++- validator/net/download-state.cpp | 10 ++- validator/net/download-state.hpp | 4 + validator/state-serializer.cpp | 44 ++++++++++ validator/state-serializer.hpp | 8 ++ validator/stats-provider.h | 105 +++++++++++++++++++++++ validator/validator.h | 8 ++ 26 files changed, 365 insertions(+), 45 deletions(-) create mode 100644 validator/stats-provider.h diff --git a/tdutils/td/utils/Time.h b/tdutils/td/utils/Time.h index ece822d4..c7795ae4 100644 --- a/tdutils/td/utils/Time.h +++ b/tdutils/td/utils/Time.h @@ -128,6 +128,10 @@ inline Timestamp &operator+=(Timestamp &a, double b) { return a; } +inline double operator-(const Timestamp &a, const Timestamp &b) { + return a.at() - b.at(); +} + template void store(const Timestamp ×tamp, StorerT &storer) { storer.store_binary(timestamp.at() - Time::now() + Clocks::system()); diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index cc7c57b3..81b8278f 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1957,7 +1957,8 @@ void ValidatorEngine::started_overlays() { void ValidatorEngine::start_validator() { validator_options_.write().set_allow_blockchain_init(config_.validators.size() > 0); - validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled); + validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled && + !state_serializer_disabled_flag_); load_collator_options(); validator_manager_ = ton::validator::ValidatorManagerFactory::create( @@ -3973,7 +3974,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateS promise.set_value(ton::create_serialize_tl_object()); return; } - validator_options_.write().set_state_serializer_enabled(query.enabled_); + validator_options_.write().set_state_serializer_enabled(query.enabled_ && !state_serializer_disabled_flag_); td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, validator_options_); config_.state_serializer_enabled = query.enabled_; @@ -4556,6 +4557,11 @@ int main(int argc, char *argv[]) { td::actor::send_closure(x, &ValidatorEngine::set_validator_telemetry_filename, s); }); }); + p.add_option( + '\0', "disable-state-serializer", + "disable persistent state serializer (similar to set-state-serializer-enabled 0 in validator console)", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_state_serializer_disabled_flag); }); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index b7abb0b1..6c2f5c4b 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -228,6 +228,7 @@ class ValidatorEngine : public td::actor::Actor { std::string validator_telemetry_filename_; bool not_all_shards_ = false; std::vector add_shard_cmds_; + bool state_serializer_disabled_flag_ = false; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -325,6 +326,9 @@ class ValidatorEngine : public td::actor::Actor { void add_shard_cmd(ton::ShardIdFull shard) { add_shard_cmds_.push_back(shard); } + void set_state_serializer_disabled_flag() { + state_serializer_disabled_flag_ = true; + } void start_up() override; ValidatorEngine() { diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index d349f9d8..8c7cde17 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -1196,6 +1196,30 @@ void ArchiveManager::set_async_mode(bool mode, td::Promise promise) { } } +void ArchiveManager::prepare_stats(td::Promise>> promise) { + std::vector> stats; + { + std::map states; + for (auto &[key, file] : perm_states_) { + BlockSeqno seqno = key.first; + auto r_stat = td::stat(db_root_ + "/archive/states/" + file.filename_short()); + if (r_stat.is_error()) { + LOG(WARNING) << "Cannot stat persistent state file " << file.filename_short() << " : " << r_stat.move_as_error(); + } else { + states[seqno] += r_stat.move_as_ok().size_; + } + } + td::StringBuilder sb; + for (auto &[seqno, size] : states) { + sb << seqno << ":" << td::format::as_size(size) << " "; + } + if (!sb.as_cslice().empty()) { + stats.emplace_back("persistent_states", sb.as_cslice().str()); + } + } + promise.set_value(std::move(stats)); +} + void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { index_->begin_transaction().ensure(); td::MultiPromise mp; diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 90fc6a0b..d919e32e 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -81,6 +81,8 @@ class ArchiveManager : public td::actor::Actor { cur_shard_split_depth_ = value; } + void prepare_stats(td::Promise>> promise); + static constexpr td::uint32 archive_size() { return 20000; } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 9dcecdb3..e86a373d 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -158,6 +158,17 @@ void CellDbIn::start_up() { }, td::Timestamp::now()); } + + { + std::string key = "stats.last_deleted_mc_seqno", value; + auto R = cell_db_->get(td::as_slice(key), value); + R.ensure(); + if (R.ok() == td::KeyValue::GetStatus::Ok) { + auto r_value = td::to_integer_safe(value); + r_value.ensure(); + last_deleted_mc_state_ = r_value.move_as_ok(); + } + } } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { @@ -452,6 +463,11 @@ void CellDbIn::gc_cont2(BlockHandle handle) { cell_db_->erase(get_key(key_hash)).ensure(); set_block(F.prev, std::move(P)); set_block(F.next, std::move(N)); + if (handle->id().is_masterchain()) { + last_deleted_mc_state_ = handle->id().seqno(); + std::string key = "stats.last_deleted_mc_seqno", value = td::to_string(last_deleted_mc_state_); + cell_db_->set(td::as_slice(key), td::as_slice(value)); + } cell_db_->commit_write_batch().ensure(); alarm_timestamp() = td::Timestamp::now(); timer_write_batch.reset(); @@ -475,9 +491,6 @@ void CellDbIn::gc_cont2(BlockHandle handle) { if (!opts_->get_disable_rocksdb_stats()) { cell_db_statistics_.gc_cell_time_.insert(timer.elapsed() * 1e6); } - if (handle->id().is_masterchain()) { - last_deleted_mc_state_ = handle->id().seqno(); - } LOG(DEBUG) << "Deleted state " << handle->id().to_str(); timer_finish.reset(); timer_all.reset(); diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index e0579d57..8d83e7a7 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -438,6 +438,7 @@ void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise promise) { void RootDb::prepare_stats(td::Promise>> promise) { auto merger = StatsMerger::create(std::move(promise)); td::actor::send_closure(cell_db_, &CellDb::prepare_stats, merger.make_promise("celldb.")); + td::actor::send_closure(archive_db_, &ArchiveManager::prepare_stats, merger.make_promise("archive.")); } void RootDb::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) { diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 32978ea5..8473cb22 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -38,6 +38,7 @@ DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt mastercha } void DownloadShardState::start_up() { + status_ = ProcessStatus(manager_, "process.download_state"); alarm_timestamp() = timeout_; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -81,6 +82,7 @@ void DownloadShardState::download_state() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, block_id_, priority_, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); } void DownloadShardState::downloaded_proof_link(td::BufferSlice data) { @@ -123,6 +125,7 @@ void DownloadShardState::checked_proof_link() { td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, masterchain_block_id_, priority_, std::move(P)); } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state"); } void DownloadShardState::download_zero_state() { @@ -152,6 +155,7 @@ void DownloadShardState::downloaded_zero_state(td::BufferSlice data) { } void DownloadShardState::downloaded_shard_state(td::BufferSlice data) { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : processing downloaded state"); auto S = create_shard_state(block_id_, data.clone()); if (S.is_error()) { fail_handler(actor_id(this), S.move_as_error()); @@ -174,6 +178,7 @@ void DownloadShardState::downloaded_shard_state(td::BufferSlice data) { } void DownloadShardState::checked_shard_state() { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state file"); LOG(WARNING) << "checked shard state " << block_id_.to_str(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); @@ -189,6 +194,7 @@ void DownloadShardState::checked_shard_state() { } void DownloadShardState::written_shard_state_file() { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state to celldb"); LOG(WARNING) << "written shard state file " << block_id_.to_str(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -198,6 +204,7 @@ void DownloadShardState::written_shard_state_file() { } void DownloadShardState::written_shard_state(td::Ref state) { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : finishing"); state_ = std::move(state); handle_->set_unix_time(state_->get_unix_time()); handle_->set_is_key_block(block_id_.is_masterchain()); diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index 02984c53..bde80aae 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -19,6 +19,7 @@ #pragma once #include "validator/interfaces/validator-manager.h" +#include "stats-provider.h" namespace ton { @@ -67,6 +68,8 @@ class DownloadShardState : public td::actor::Actor { td::BufferSlice data_; td::Ref state_; + + ProcessStatus status_; }; } // namespace validator diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index a781968d..ce21bc5e 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -50,7 +50,7 @@ class Collator final : public td::actor::Actor { using LtCellRef = block::LtCellRef; using NewOutMsg = block::NewOutMsg; const ShardIdFull shard_; - ton::BlockId new_id; + ton::BlockId new_id{workchainInvalid, 0, 0}; bool busy_{false}; bool before_split_{false}; bool after_split_{false}; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index d3378cd8..d5c41853 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -352,6 +352,8 @@ bool Collator::fatal_error(td::Status error) { attempt_idx_ + 1); } else { main_promise(std::move(error)); + td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, BlockIdExt{new_id, RootHash::zero(), FileHash::zero()}, + work_timer_.elapsed(), cpu_work_timer_.elapsed(), td::optional{}); } busy_ = false; } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 723dbfe9..83f39c45 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -85,19 +85,13 @@ void LiteQuery::abort_query(td::Status reason) { if (acc_state_promise_) { acc_state_promise_.set_error(std::move(reason)); } else if (promise_) { + td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_ ? query_obj_->get_id() : 0, + false); promise_.set_error(std::move(reason)); } stop(); } -void LiteQuery::abort_query_ext(td::Status reason, std::string comment) { - LOG(INFO) << "aborted liteserver query: " << comment << " : " << reason.to_string(); - if (promise_) { - promise_.set_error(reason.move_as_error_prefix(comment + " : ")); - } - stop(); -} - bool LiteQuery::fatal_error(td::Status error) { abort_query(std::move(error)); return false; @@ -120,6 +114,8 @@ bool LiteQuery::finish_query(td::BufferSlice result, bool skip_cache_update) { td::actor::send_closure(cache_, &LiteServerCache::update, cache_key_, result.clone()); } if (promise_) { + td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_ ? query_obj_->get_id() : 0, + true); promise_.set_result(std::move(result)); stop(); return true; @@ -139,7 +135,6 @@ void LiteQuery::start_up() { auto F = fetch_tl_object(query_, true); if (F.is_error()) { - td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, 0); // unknown abort_query(F.move_as_error()); return; } @@ -192,7 +187,6 @@ bool LiteQuery::use_cache() { } void LiteQuery::perform() { - td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_->get_id()); lite_api::downcast_call( *query_obj_, td::overloaded( diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 447e1dad..fc873533 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -97,7 +97,6 @@ class LiteQuery : public td::actor::Actor { bool fatal_error(std::string err_msg, int err_code = -400); bool fatal_error(int err_code, std::string err_msg = ""); void abort_query(td::Status reason); - void abort_query_ext(td::Status reason, std::string err_msg); bool finish_query(td::BufferSlice result, bool skip_cache_update = false); void alarm() override; void start_up() override; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 583b1d86..31c30e90 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -115,7 +115,7 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { error = error_ctx() + error; LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { - record_stats(); + record_stats(false); errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -153,7 +153,7 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) error = error_ctx() + error; LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { - record_stats(); + record_stats(false); errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -176,7 +176,7 @@ bool ValidateQuery::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string(); if (main_promise) { - record_stats(); + record_stats(false); auto c = error.code(); if (c <= -667 && c >= -670) { errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str() @@ -234,7 +234,7 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) { */ void ValidateQuery::finish_query() { if (main_promise) { - record_stats(); + record_stats(true); LOG(WARNING) << "validate query done"; main_promise.set_result(now_); } @@ -6928,13 +6928,13 @@ void ValidateQuery::written_candidate() { /** * Sends validation work time to manager. */ -void ValidateQuery::record_stats() { +void ValidateQuery::record_stats(bool success) { double work_time = work_timer_.elapsed(); double cpu_work_time = cpu_work_timer_.elapsed(); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time, - cpu_work_time); + cpu_work_time, success); } } // namespace validator diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 98cd2493..90c368ff 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -400,7 +400,7 @@ class ValidateQuery : public td::actor::Actor { td::Timer work_timer_{true}; td::ThreadCpuTimer cpu_work_timer_{true}; - void record_stats(); + void record_stats(bool success); }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 20d4bd62..00fb77e1 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -205,13 +205,13 @@ class ValidatorManager : public ValidatorManagerInterface { td::optional shard, td::Promise> promise) = 0; - virtual void add_lite_query_stats(int lite_query_id) { + virtual void add_lite_query_stats(int lite_query_id, bool success) { } virtual void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) { + td::optional stats) { } - virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, bool success) { } virtual void add_persistent_state_description(td::Ref desc) = 0; diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index c2944b25..6f304680 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -32,6 +32,8 @@ namespace ton { namespace validator { void ValidatorManagerMasterchainReiniter::start_up() { + status_ = ProcessStatus(manager_, "process.initial_sync"); + status_.set_status(PSTRING() << "starting, init block seqno " << block_id_.seqno()); LOG(INFO) << "init_block_id=" << block_id_; CHECK(block_id_.is_masterchain()); CHECK(block_id_.id.shard == shardIdAll); @@ -58,6 +60,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han key_blocks_.push_back(handle_); if (opts_->initial_sync_disabled()) { + status_.set_status(PSTRING() << "downloading masterchain state " << handle_->id().seqno()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_masterchain_state); @@ -181,6 +184,7 @@ void ValidatorManagerMasterchainReiniter::got_next_key_blocks(std::vector(key_blocks_.size()); key_blocks_.resize(key_blocks_.size() + vec.size(), nullptr); @@ -247,6 +251,7 @@ void ValidatorManagerMasterchainReiniter::choose_masterchain_state() { } void ValidatorManagerMasterchainReiniter::download_masterchain_state() { + status_.set_status(PSTRING() << "downloading masterchain state " << block_id_.seqno()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "failed to download masterchain state: " << R.move_as_error(); @@ -274,6 +279,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Ref("shardclient", opts_, handle_, state_, manager_, std::move(P)); + status_.set_status(PSTRING() << "downloading all shard states, mc seqno " << block_id_.seqno()); } void ValidatorManagerMasterchainReiniter::downloaded_all_shards() { diff --git a/validator/manager-init.hpp b/validator/manager-init.hpp index 7dce4e47..901b826b 100644 --- a/validator/manager-init.hpp +++ b/validator/manager-init.hpp @@ -27,6 +27,8 @@ #include "manager-init.h" +#include + namespace ton { namespace validator { @@ -77,6 +79,8 @@ class ValidatorManagerMasterchainReiniter : public td::actor::Actor { td::uint32 pending_ = 0; td::actor::ActorOwn client_; + + ProcessStatus status_; }; class ValidatorManagerMasterchainStarter : public td::actor::Actor { diff --git a/validator/manager.cpp b/validator/manager.cpp index 068ea5eb..8dce764d 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -451,11 +451,9 @@ void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Prom promise = [self = this, wc, addr, promise = std::move(promise), SelfId = actor_id(this)](td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - td::actor::send_lambda(SelfId, [=, promise = std::move(promise), message = R.move_as_ok()]() mutable { + td::actor::send_lambda(SelfId, [=, promise = std::move(promise), R = std::move(R)]() mutable { + ++(R.is_ok() ? self->total_check_ext_messages_ok_ : self->total_check_ext_messages_error_); + TRY_RESULT_PROMISE(promise, message, std::move(R)); if (self->checked_ext_msg_counter_.inc_msg_count(wc, addr) > max_ext_msg_per_addr()) { promise.set_error( td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); @@ -2131,7 +2129,7 @@ void ValidatorManagerImpl::update_shards() { } } - bool validating_masterchain = false; + active_validator_groups_master_ = active_validator_groups_shard_ = 0; if (allow_validate_) { for (auto &desc : new_shards) { auto shard = desc.first; @@ -2148,9 +2146,7 @@ void ValidatorManagerImpl::update_shards() { auto validator_id = get_validator(shard, val_set); if (!validator_id.is_zero()) { - if (shard.is_masterchain()) { - validating_masterchain = true; - } + ++(shard.is_masterchain() ? active_validator_groups_master_ : active_validator_groups_shard_); auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); if (force_recover) { @@ -2845,8 +2841,8 @@ void ValidatorManagerImpl::prepare_stats(td::Promiseid().to_str()); vec.emplace_back("rotatemasterchainblock", last_rotate_block_id_.to_str()); //vec.emplace_back("shardclientmasterchainseqno", td::to_string(min_confirmed_masterchain_seqno_)); - vec.emplace_back("stateserializermasterchainseqno", td::to_string(state_serializer_masterchain_seqno_)); } + td::NamedThreadSafeCounter::get_default().for_each([&](auto key, auto value) { vec.emplace_back("counter." + key, PSTRING() << value); }); @@ -2864,9 +2860,48 @@ void ValidatorManagerImpl::prepare_stats(td::Promiseget_state_serializer_enabled(); + if (is_validator() && last_masterchain_state_->get_global_id() == -239) { + serializer_enabled = false; + } + vec.emplace_back("stateserializerenabled", serializer_enabled ? "true" : "false"); + merger.make_promise("").set_value(std::move(vec)); + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::prepare_stats, merger.make_promise("")); + } + td::actor::send_closure(db_, &Db::prepare_stats, merger.make_promise("db.")); + for (auto &[_, p] : stats_providers_) { + p.second(merger.make_promise(p.first)); + } } void ValidatorManagerImpl::prepare_perf_timer_stats(td::Promise> promise) { @@ -3353,17 +3388,28 @@ td::actor::ActorOwn ValidatorManagerFactory::create( } void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) { + td::optional stats) { + if (!stats) { + ++(block_id.is_masterchain() ? total_collated_blocks_master_error_ : total_collated_blocks_shard_error_); + return; + } auto &record = new_block_stats_record(block_id); record.collator_work_time_ = work_time; record.collator_cpu_work_time_ = cpu_work_time; - record.collator_stats_ = std::move(stats); + record.collator_stats_ = std::move(stats.value()); + ++(block_id.is_masterchain() ? total_collated_blocks_master_ok_ : total_collated_blocks_shard_ok_); } -void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { +void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + bool success) { auto &record = new_block_stats_record(block_id); record.validator_work_time_ = work_time; record.validator_cpu_work_time_ = cpu_work_time; + if (success) { + ++(block_id.is_masterchain() ? total_validated_blocks_master_ok_ : total_validated_blocks_shard_ok_); + } else { + ++(block_id.is_masterchain() ? total_validated_blocks_master_error_ : total_validated_blocks_shard_error_); + } } ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) { @@ -3377,6 +3423,16 @@ ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_ return recorded_block_stats_[block_id]; } +void ValidatorManagerImpl::register_stats_provider( + td::uint64 idx, std::string prefix, + std::function>>)> callback) { + stats_providers_[idx] = {std::move(prefix), std::move(callback)}; +} + +void ValidatorManagerImpl::unregister_stats_provider(td::uint64 idx) { + stats_providers_.erase(idx); +} + size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { before_query(); auto it1 = counter_cur_.find({wc, addr}); diff --git a/validator/manager.hpp b/validator/manager.hpp index 519cab12..9e54c3f3 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -655,8 +655,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::optional shard, td::Promise> promise) override; - void add_lite_query_stats(int lite_query_id) override { + void add_lite_query_stats(int lite_query_id, bool success) override { ++ls_stats_[lite_query_id]; + ++(success ? total_ls_queries_ok_ : total_ls_queries_error_)[lite_query_id]; } private: @@ -747,6 +748,16 @@ class ValidatorManagerImpl : public ValidatorManager { std::map ls_stats_; // lite_api ID -> count, 0 for unknown td::uint32 ls_stats_check_ext_messages_{0}; + UnixTime started_at_ = (UnixTime)td::Clocks::system(); + std::map total_ls_queries_ok_, total_ls_queries_error_; // lite_api ID -> count, 0 for unknown + td::uint64 total_check_ext_messages_ok_{0}, total_check_ext_messages_error_{0}; + td::uint64 total_collated_blocks_master_ok_{0}, total_collated_blocks_master_error_{0}; + td::uint64 total_validated_blocks_master_ok_{0}, total_validated_blocks_master_error_{0}; + td::uint64 total_collated_blocks_shard_ok_{0}, total_collated_blocks_shard_error_{0}; + td::uint64 total_validated_blocks_shard_ok_{0}, total_validated_blocks_shard_error_{0}; + + size_t active_validator_groups_master_{0}, active_validator_groups_shard_{0}; + td::actor::ActorOwn candidates_buffer_; struct RecordedBlockStats { @@ -760,16 +771,25 @@ class ValidatorManagerImpl : public ValidatorManager { std::queue recorded_block_stats_lru_; void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) override; - void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override; + td::optional stats) override; + void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, bool success) override; RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); + void register_stats_provider( + td::uint64 idx, std::string prefix, + std::function>>)> callback) override; + void unregister_stats_provider(td::uint64 idx) override; + std::map> validator_telemetry_; void init_validator_telemetry(); std::map> persistent_state_descriptions_; std::map> persistent_state_blocks_; + + std::map>>)>>> + stats_providers_; }; } // namespace validator diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 2b373ef3..6735a2b5 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -70,6 +70,7 @@ void DownloadState::finish_query() { } void DownloadState::start_up() { + status_ = ProcessStatus(validator_manager_, "process.download_state_net"); alarm_timestamp() = timeout_; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, @@ -190,6 +191,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { td::Timestamp::in(3.0), std::move(P)); } })); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : 0 bytes, 0B/s"); } void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 requested_size) { @@ -198,14 +200,18 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques parts_.push_back(std::move(data)); double elapsed = prev_logged_timer_.elapsed(); - if (elapsed > 10.0) { + if (elapsed > 5.0) { prev_logged_timer_ = td::Timer(); + auto speed = (td::uint64)((double)(sum_ - prev_logged_sum_) / elapsed); LOG(WARNING) << "downloading state " << block_id_.to_str() << ": " << td::format::as_size(sum_) << " (" - << td::format::as_size((td::uint64)(double(sum_ - prev_logged_sum_) / elapsed)) << "/s)"; + << td::format::as_size(speed) << "/s)"; + status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, " << td::format::as_size(speed) + << "/s"); prev_logged_sum_ = sum_; } if (last_part) { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, finishing"); td::BufferSlice res{td::narrow_cast(sum_)}; auto S = res.as_slice(); for (auto &p : parts_) { diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index 19c44beb..470c5431 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -23,6 +23,8 @@ #include "validator/validator.h" #include "adnl/adnl-ext-client.h" +#include + namespace ton { namespace validator { @@ -75,6 +77,8 @@ class DownloadState : public td::actor::Actor { td::uint64 prev_logged_sum_ = 0; td::Timer prev_logged_timer_; + + ProcessStatus status_; }; } // namespace fullnode diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index b693232b..bc3d7b5e 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -58,6 +58,12 @@ void AsyncStateSerializer::got_self_state(AsyncSerializerState state) { }); td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, last_block_id_, true, std::move(P)); } + + inited_block_id_ = true; + for (auto& promise : wait_init_block_id_) { + promise.set_value(td::Unit()); + } + wait_init_block_id_.clear(); } void AsyncStateSerializer::got_init_handle(BlockHandle handle) { @@ -186,6 +192,9 @@ void AsyncStateSerializer::next_iteration() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_previous_state_files); }, td::Timestamp::in(delay)); + current_status_ = PSTRING() << "delay before serializing seqno=" << masterchain_handle_->id().seqno() << " " + << (int)delay << "s"; + current_status_ts_ = td::Timestamp::now(); return; } if (next_idx_ < shards_.size()) { @@ -379,9 +388,14 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, masterchain_handle_->id(), masterchain_handle_->id(), write_data, std::move(P)); + + current_status_ = PSTRING() << "serializing masterchain state " << state->get_block_id().id.to_str(); + current_status_ts_ = td::Timestamp::now(); } void AsyncStateSerializer::stored_masterchain_state() { + current_status_ = "pending"; + current_status_ts_ = {}; LOG(ERROR) << "finished serializing masterchain state " << masterchain_handle_->id().id.to_str(); running_ = false; next_iteration(); @@ -444,9 +458,14 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Refid(), masterchain_handle_->id(), write_data, std::move(P)); + current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " " + << state->get_block_id().id.to_str(); + current_status_ts_ = td::Timestamp::now(); } void AsyncStateSerializer::fail_handler(td::Status reason) { + current_status_ = PSTRING() << "pending, " << reason; + current_status_ts_ = {}; VLOG(VALIDATOR_NOTICE) << "failure: " << reason; attempt_++; delay_action( @@ -460,6 +479,8 @@ void AsyncStateSerializer::fail_handler_cont() { } void AsyncStateSerializer::success_handler() { + current_status_ = "pending"; + current_status_ts_ = {}; running_ = false; next_iteration(); } @@ -478,6 +499,29 @@ void AsyncStateSerializer::auto_disable_serializer(bool disabled) { } } +void AsyncStateSerializer::prepare_stats(td::Promise>> promise) { + if (!inited_block_id_) { + wait_init_block_id_.push_back( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &AsyncStateSerializer::prepare_stats, std::move(promise)); + }); + return; + } + std::vector> vec; + vec.emplace_back("stateserializermasterchainseqno", td::to_string(last_block_id_.seqno())); + td::StringBuilder sb; + sb << current_status_; + if (current_status_ts_) { + sb << " (started " << (int)(td::Timestamp::now() - current_status_ts_) << "s ago)"; + } + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { + sb << " (disabled)"; + } + vec.emplace_back("stateserializerstatus", sb.as_cslice().str()); + promise.set_result(std::move(vec)); +} + bool AsyncStateSerializer::need_serialize(BlockHandle handle) { if (handle->id().id.seqno == 0 || !handle->is_key_block()) { return false; diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 1e7f5c9c..406ac350 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -36,6 +36,9 @@ class AsyncStateSerializer : public td::actor::Actor { UnixTime last_key_block_ts_ = 0; bool saved_to_db_ = true; + bool inited_block_id_ = false; + std::vector> wait_init_block_id_; + td::Ref opts_; bool auto_disabled_ = false; td::CancellationTokenSource cancellation_token_source_; @@ -95,6 +98,8 @@ class AsyncStateSerializer : public td::actor::Actor { promise.set_result(last_block_id_.id.seqno); } + void prepare_stats(td::Promise>> promise); + void update_last_known_key_block_ts(UnixTime ts) { last_known_key_block_ts_ = std::max(last_known_key_block_ts_, ts); } @@ -111,6 +116,9 @@ class AsyncStateSerializer : public td::actor::Actor { void update_options(td::Ref opts); void auto_disable_serializer(bool disabled); + + std::string current_status_ = "pending"; + td::Timestamp current_status_ts_ = td::Timestamp::never(); }; } // namespace validator diff --git a/validator/stats-provider.h b/validator/stats-provider.h new file mode 100644 index 00000000..e0a7f565 --- /dev/null +++ b/validator/stats-provider.h @@ -0,0 +1,105 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "validator.h" +#include "common/AtomicRef.h" + +#include + +namespace ton { + +namespace validator { + +class StatsProvider { + public: + StatsProvider() = default; + StatsProvider(td::actor::ActorId manager, std::string prefix, + std::function>>)> callback) + : inited_(true), manager_(std::move(manager)) { + static std::atomic cur_idx{0}; + idx_ = cur_idx.fetch_add(1); + td::actor::send_closure(manager_, &ValidatorManagerInterface::register_stats_provider, idx_, std::move(prefix), + std::move(callback)); + } + StatsProvider(const StatsProvider&) = delete; + StatsProvider(StatsProvider&& other) noexcept + : inited_(other.inited_), idx_(other.idx_), manager_(std::move(other.manager_)) { + other.inited_ = false; + } + ~StatsProvider() { + if (inited_) { + td::actor::send_closure(manager_, &ValidatorManagerInterface::unregister_stats_provider, idx_); + } + } + + StatsProvider& operator=(const StatsProvider&) = delete; + StatsProvider& operator=(StatsProvider&& other) noexcept { + if (this != &other) { + inited_ = other.inited_; + idx_ = other.idx_; + manager_ = std::move(other.manager_); + other.inited_ = false; + } + return *this; + } + + bool inited() const { + return inited_; + } + + private: + bool inited_ = false; + td::uint64 idx_ = 0; + td::actor::ActorId manager_; +}; + +class ProcessStatus { + public: + ProcessStatus() = default; + ProcessStatus(td::actor::ActorId manager, std::string name) + : stats_provider_(std::move(manager), std::move(name), [value = value_](auto promise) { + auto status = value->load(); + if (status.is_null()) { + promise.set_error(td::Status::Error("empty")); + return; + } + std::vector> vec; + vec.emplace_back("", *status); + promise.set_value(std::move(vec)); + }) { + } + ProcessStatus(const ProcessStatus&) = delete; + ProcessStatus(ProcessStatus&& other) noexcept = default; + ProcessStatus& operator=(const ProcessStatus&) = delete; + ProcessStatus& operator=(ProcessStatus&& other) noexcept = default; + + void set_status(std::string s) { + if (!value_) { + return; + } + value_->store(td::Ref>(true, std::move(s))); + } + + private: + std::shared_ptr>> value_ = std::make_shared>>(); + StatsProvider stats_provider_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/validator.h b/validator/validator.h index 73065aa9..42b3f69a 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -20,6 +20,7 @@ #include #include +#include #include "td/actor/actor.h" @@ -292,6 +293,13 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) = 0; virtual void update_options(td::Ref opts) = 0; + + virtual void register_stats_provider( + td::uint64 idx, std::string prefix, + std::function>>)> callback) { + } + virtual void unregister_stats_provider(td::uint64 idx) { + } }; } // namespace validator From aca51a8dae297c6b853860af9f0565be3d25527d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 17 Feb 2025 10:14:12 +0300 Subject: [PATCH 03/10] Don't check external messages if out of sync --- validator/manager.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/validator/manager.cpp b/validator/manager.cpp index 8dce764d..2cc38211 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -430,6 +430,10 @@ void ValidatorManagerImpl::add_external_message(td::Ref msg, int pri ext_messages_hashes_[id.hash] = {priority, id}; } void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise> promise) { + if (!started_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "node not synced")); + return; + } auto state = do_get_last_liteserver_state(); if (state.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); From 04f2bc13605e0189346f041baa515fbccab729f4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 19 Feb 2025 12:44:50 +0300 Subject: [PATCH 04/10] Fix downloading persistent states in WaitBlockState --- validator/manager.cpp | 25 ++++++++++++++++--------- validator/manager.hpp | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 2cc38211..b0ac5409 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -700,11 +700,10 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = - td::actor::create_actor("waitstate", handle, priority, actor_id(this), - td::Timestamp::at(timeout.at() + 10.0), std::move(P), - get_block_persistent_state(handle->id())) - .release(); + auto id = td::actor::create_actor("waitstate", handle, priority, actor_id(this), + td::Timestamp::at(timeout.at() + 10.0), std::move(P), + get_block_persistent_state_to_download(handle->id())) + .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); } @@ -1150,9 +1149,10 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, X.second, actor_id(this), X.first, - std::move(P), get_block_persistent_state(handle->id())) - .release(); + auto id = + td::actor::create_actor("waitstate", handle, X.second, actor_id(this), X.first, + std::move(P), get_block_persistent_state_to_download(handle->id())) + .release(); it->second.actor_ = id; return; } @@ -3375,11 +3375,18 @@ void ValidatorManagerImpl::got_persistent_state_descriptions(std::vector ValidatorManagerImpl::get_block_persistent_state(BlockIdExt block_id) { +td::Ref ValidatorManagerImpl::get_block_persistent_state_to_download(BlockIdExt block_id) { + if (block_id.is_masterchain()) { + return {}; + } auto it = persistent_state_blocks_.find(block_id); if (it == persistent_state_blocks_.end()) { return {}; } + if (it->second->masterchain_id.seqno() + 16 >= min_confirmed_masterchain_seqno_) { + // Do not download persistent states during ordinary shard client sync + return {}; + } return it->second; } diff --git a/validator/manager.hpp b/validator/manager.hpp index 9e54c3f3..418deb35 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -734,7 +734,7 @@ class ValidatorManagerImpl : public ValidatorManager { void got_persistent_state_descriptions(std::vector> descs); void add_persistent_state_description_impl(td::Ref desc); - td::Ref get_block_persistent_state(BlockIdExt block_id); + td::Ref get_block_persistent_state_to_download(BlockIdExt block_id); private: bool need_monitor(ShardIdFull shard) const { From 8a08bf67a24519fcaef293d21266612c673761fa Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 20 Feb 2025 17:32:24 +0300 Subject: [PATCH 05/10] Experimental flags for speeding up broadcasts --- catchain/catchain-receiver.cpp | 6 ++-- overlay/overlay-fec.cpp | 12 ++++--- overlay/overlay-fec.hpp | 5 +-- overlay/overlay.cpp | 6 ++-- overlay/overlays.h | 1 + ton/ton-types.h | 1 + validator-engine/validator-engine.cpp | 43 ++++++++++++++++++++++++- validator-engine/validator-engine.hpp | 12 +++++++ validator/full-node-private-overlay.cpp | 16 ++++++--- validator/full-node-private-overlay.hpp | 16 ++++----- validator/full-node-shard.cpp | 13 ++++---- validator/full-node-shard.h | 2 +- validator/full-node-shard.hpp | 6 ++-- validator/full-node.cpp | 16 ++++----- validator/full-node.h | 8 ++++- validator/full-node.hpp | 4 +-- validator/validator-group.cpp | 1 + validator/validator-options.hpp | 7 ++++ validator/validator.h | 2 ++ 19 files changed, 131 insertions(+), 46 deletions(-) diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index edef9065..a6160383 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -526,10 +526,12 @@ void CatChainReceiverImpl::start_up() { for (td::uint32 i = 0; i < get_sources_cnt(); i++) { root_keys.emplace(get_source(i)->get_hash(), OVERLAY_MAX_ALLOWED_PACKET_SIZE); } - td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay, + overlay::OverlayOptions overlay_options; + overlay_options.broadcast_speed_multiplier_ = opts_.broadcast_speed_multiplier; + td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay_ex, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}, - R"({ "type": "catchain" })"); + R"({ "type": "catchain" })", std::move(overlay_options)); CHECK(root_block_); diff --git a/overlay/overlay-fec.cpp b/overlay/overlay-fec.cpp index b29fce22..817d3b7c 100644 --- a/overlay/overlay-fec.cpp +++ b/overlay/overlay-fec.cpp @@ -32,7 +32,7 @@ void OverlayOutboundFecBroadcast::alarm() { fec_type_.size(), flags_, std::move(X.data), X.id, fec_type_, date_); } - alarm_timestamp() = td::Timestamp::in(0.010); + alarm_timestamp() = td::Timestamp::in(delay_); if (seqno_ >= to_send_) { stop(); @@ -46,8 +46,9 @@ void OverlayOutboundFecBroadcast::start_up() { OverlayOutboundFecBroadcast::OverlayOutboundFecBroadcast(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, - PublicKeyHash local_id) + PublicKeyHash local_id, double speed_multiplier) : flags_(flags) { + delay_ /= speed_multiplier; CHECK(data.size() <= (1 << 27)); local_id_ = local_id; overlay_ = std::move(overlay); @@ -63,9 +64,10 @@ OverlayOutboundFecBroadcast::OverlayOutboundFecBroadcast(td::BufferSlice data, t } td::actor::ActorId OverlayOutboundFecBroadcast::create( - td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, PublicKeyHash local_id) { - return td::actor::create_actor(td::actor::ActorOptions().with_name("bcast"), - std::move(data), flags, overlay, local_id) + td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, PublicKeyHash local_id, + double speed_multiplier) { + return td::actor::create_actor( + td::actor::ActorOptions().with_name("bcast"), std::move(data), flags, overlay, local_id, speed_multiplier) .release(); } diff --git a/overlay/overlay-fec.hpp b/overlay/overlay-fec.hpp index a9cc3634..b72e830e 100644 --- a/overlay/overlay-fec.hpp +++ b/overlay/overlay-fec.hpp @@ -37,6 +37,7 @@ class OverlayOutboundFecBroadcast : public td::actor::Actor { PublicKeyHash local_id_; Overlay::BroadcastDataHash data_hash_; td::uint32 flags_ = 0; + double delay_ = 0.010; td::int32 date_; std::unique_ptr encoder_; td::actor::ActorId overlay_; @@ -45,9 +46,9 @@ class OverlayOutboundFecBroadcast : public td::actor::Actor { public: static td::actor::ActorId create(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, - PublicKeyHash local_id); + PublicKeyHash local_id, double speed_multiplier = 1.0); OverlayOutboundFecBroadcast(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, - PublicKeyHash local_id); + PublicKeyHash local_id, double speed_multiplier = 1.0); void alarm() override; void start_up() override; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 429c6a9c..30a40b1c 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -63,7 +63,7 @@ td::actor::ActorOwn Overlay::create_private( return td::actor::create_actor( overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::FixedMemberList, std::move(nodes), std::vector(), OverlayMemberCertificate{}, - std::move(callback), std::move(rules), std::move(scope)); + std::move(callback), std::move(rules), std::move(scope), std::move(opts)); } td::actor::ActorOwn Overlay::create_semiprivate( @@ -99,6 +99,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor overlay_id_ = id_full_.compute_short_id(); frequent_dht_lookup_ = opts_.frequent_dht_lookup_; peer_list_.local_member_flags_ = opts_.local_overlay_member_flags_; + opts_.broadcast_speed_multiplier_ = std::max(opts_.broadcast_speed_multiplier_, 1e-9); VLOG(OVERLAY_INFO) << this << ": creating"; @@ -490,7 +491,8 @@ void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; return; } - OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as); + OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as, + opts_.broadcast_speed_multiplier_); } void OverlayImpl::print(td::StringBuilder &sb) { diff --git a/overlay/overlays.h b/overlay/overlays.h index c0385fc7..5eb63b13 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -269,6 +269,7 @@ struct OverlayOptions { td::uint32 nodes_to_send_ = 4; td::uint32 propagate_broadcast_to_ = 5; td::uint32 default_permanent_members_flags_ = 0; + double broadcast_speed_multiplier_ = 1.0; }; class Overlays : public td::actor::Actor { diff --git a/ton/ton-types.h b/ton/ton-types.h index 2447a8c5..c7aff644 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -493,6 +493,7 @@ struct CatChainOptions { td::uint64 max_block_height_coeff = 0; bool debug_disable_db = false; + double broadcast_speed_multiplier = 1.0; }; struct ValidatorSessionConfig { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 81b8278f..2ea04e18 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1504,6 +1504,7 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_.write().set_hardforks(std::move(h)); validator_options_.write().set_fast_state_serializer_enabled(fast_state_serializer_enabled_); + validator_options_.write().set_catchain_broadcast_speed_multiplier(broadcast_speed_multiplier_catchain_); return td::Status::OK(); } @@ -2004,9 +2005,13 @@ void ValidatorEngine::start_full_node() { R.ensure(); td::actor::send_closure(SelfId, &ValidatorEngine::started_full_node); }); + ton::validator::fullnode::FullNodeOptions full_node_options{ + .config_ = config_.full_node_config, + .public_broadcast_speed_multiplier_ = broadcast_speed_multiplier_public_, + .private_broadcast_speed_multiplier_ = broadcast_speed_multiplier_private_}; full_node_ = ton::validator::fullnode::FullNode::create( short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, - config_.full_node_config, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), + full_node_options, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); for (auto &v : config_.validators) { @@ -4562,6 +4567,42 @@ int main(int argc, char *argv[]) { "disable persistent state serializer (similar to set-state-serializer-enabled 0 in validator console)", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_state_serializer_disabled_flag); }); }); + p.add_checked_option( + '\0', "broadcast-speed-catchain", + "multiplier for broadcast speed in catchain overlays (experimental, default is 1.0, which is ~300 KB/s)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v <= 0.0) { + return td::Status::Error("broadcast-speed-catchain should be positive"); + } + acts.push_back( + [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_catchain, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + '\0', "broadcast-speed-public", + "multiplier for broadcast speed in public shard overlays (experimental, default is 1.0, which is ~300 KB/s)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v <= 0.0) { + return td::Status::Error("broadcast-speed-public should be positive"); + } + acts.push_back( + [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_public, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + '\0', "broadcast-speed-private", + "multiplier for broadcast speed in private block overlays (experimental, default is 1.0, which is ~300 KB/s)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v <= 0.0) { + return td::Status::Error("broadcast-speed-private should be positive"); + } + acts.push_back( + [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_private, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 6c2f5c4b..e0dc91f1 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -229,6 +229,9 @@ class ValidatorEngine : public td::actor::Actor { bool not_all_shards_ = false; std::vector add_shard_cmds_; bool state_serializer_disabled_flag_ = false; + double broadcast_speed_multiplier_catchain_ = 1.0; + double broadcast_speed_multiplier_public_ = 1.0; + double broadcast_speed_multiplier_private_ = 1.0; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -329,6 +332,15 @@ class ValidatorEngine : public td::actor::Actor { void set_state_serializer_disabled_flag() { state_serializer_disabled_flag_ = true; } + void set_broadcast_speed_multiplier_catchain(double value) { + broadcast_speed_multiplier_catchain_ = value; + } + void set_broadcast_speed_multiplier_public(double value) { + broadcast_speed_multiplier_public_ = value; + } + void set_broadcast_speed_multiplier_private(double value) { + broadcast_speed_multiplier_private_ = value; + } void start_up() override; ValidatorEngine() { diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 1acfbd4e..f86323fc 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -264,8 +264,11 @@ void FullNodePrivateBlockOverlay::init() { overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), overlay::CertificateFlags::AllowFec | overlay::CertificateFlags::Trusted, {}}; - td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), - nodes_, std::make_unique(actor_id(this)), rules, R"({ "type": "private-blocks" })"); + overlay::OverlayOptions overlay_options; + overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; + td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), + nodes_, std::make_unique(actor_id(this)), rules, R"({ "type": "private-blocks" })", + overlay_options); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); @@ -366,7 +369,7 @@ void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice } void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) { - if (!inited_ || config_.ext_messages_broadcast_disabled_) { + if (!inited_ || opts_.config_.ext_messages_broadcast_disabled_) { return; } VLOG(FULL_NODE_DEBUG) << "Sending external message to custom overlay \"" << name_ << "\""; @@ -472,10 +475,13 @@ void FullNodeCustomOverlay::init() { authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); } overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; + overlay::OverlayOptions overlay_options; + overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; td::actor::send_closure( - overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), nodes_, + overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), nodes_, std::make_unique(actor_id(this)), rules, - PSTRING() << R"({ "type": "custom-overlay", "name": ")" << td::format::Escaped{name_} << R"(" })"); + PSTRING() << R"({ "type": "custom-overlay", "name": ")" << td::format::Escaped{name_} << R"(" })", + overlay_options); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 1e29f8c9..70e196ea 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -50,14 +50,14 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void collect_validator_telemetry(std::string filename); void set_config(FullNodeConfig config) { - config_ = std::move(config); + opts_.config_ = std::move(config); } void start_up() override; void tear_down() override; FullNodePrivateBlockOverlay(adnl::AdnlNodeIdShort local_id, std::vector nodes, - FileHash zero_state_file_hash, FullNodeConfig config, + FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, @@ -66,7 +66,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { : local_id_(local_id) , nodes_(std::move(nodes)) , zero_state_file_hash_(zero_state_file_hash) - , config_(config) + , opts_(opts) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) @@ -80,7 +80,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { adnl::AdnlNodeIdShort local_id_; std::vector nodes_; FileHash zero_state_file_hash_; - FullNodeConfig config_; + FullNodeOptions opts_; bool enable_compression_ = true; td::actor::ActorId keyring_; @@ -126,14 +126,14 @@ class FullNodeCustomOverlay : public td::actor::Actor { td::BufferSlice data); void set_config(FullNodeConfig config) { - config_ = std::move(config); + opts_.config_ = std::move(config); } void start_up() override; void tear_down() override; FullNodeCustomOverlay(adnl::AdnlNodeIdShort local_id, CustomOverlayParams params, FileHash zero_state_file_hash, - FullNodeConfig config, td::actor::ActorId keyring, + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, @@ -144,7 +144,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { , msg_senders_(std::move(params.msg_senders_)) , block_senders_(std::move(params.block_senders_)) , zero_state_file_hash_(zero_state_file_hash) - , config_(config) + , opts_(opts) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) @@ -161,7 +161,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { std::map msg_senders_; std::set block_senders_; FileHash zero_state_file_hash_; - FullNodeConfig config_; + FullNodeOptions opts_; td::actor::ActorId keyring_; td::actor::ActorId adnl_; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 7d33a195..ac0eb768 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -105,6 +105,7 @@ void FullNodeShardImpl::create_overlay() { }; overlay::OverlayOptions opts; opts.announce_self_ = active_; + opts.broadcast_speed_multiplier_ = opts_.public_broadcast_speed_multiplier_; td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_ex, adnl_id_, overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() @@ -132,7 +133,7 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad if (!processed_ext_msg_broadcasts_.insert(hash).second) { return promise.set_error(td::Status::Error("duplicate external message broadcast")); } - if (config_.ext_messages_broadcast_disabled_) { + if (opts_.config_.ext_messages_broadcast_disabled_) { promise.set_error(td::Status::Error("rebroadcasting external messages is disabled")); promise = [manager = validator_manager_, message = q->message_->data_.clone()](td::Result R) mutable { if (R.is_ok()) { @@ -850,7 +851,7 @@ void FullNodeShardImpl::send_ihr_message(td::BufferSlice data) { } void FullNodeShardImpl::send_external_message(td::BufferSlice data) { - if (config_.ext_messages_broadcast_disabled_) { + if (opts_.config_.ext_messages_broadcast_disabled_) { return; } if (!client_.empty()) { @@ -1367,7 +1368,7 @@ void FullNodeShardImpl::get_stats_extra(td::Promise promise) { } FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, FullNodeConfig config, + FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, @@ -1387,17 +1388,17 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , client_(client) , full_node_(full_node) , active_(active) - , config_(config) { + , opts_(opts) { } td::actor::ActorOwn FullNodeShard::create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, td::actor::ActorId full_node, bool active) { return td::actor::create_actor(PSTRING() << "tonnode" << shard.to_str(), shard, local_id, adnl_id, - zero_state_file_hash, config, keyring, adnl, rldp, rldp2, overlays, + zero_state_file_hash, opts, keyring, adnl, rldp, rldp2, overlays, validator_manager, client, full_node, active); } diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 16945325..5898db80 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -76,7 +76,7 @@ class FullNodeShard : public td::actor::Actor { static td::actor::ActorOwn create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, td::actor::ActorId full_node, bool active); diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 86748134..fb3eef76 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -86,7 +86,7 @@ class FullNodeShardImpl : public FullNodeShard { void set_active(bool active) override; void set_config(FullNodeConfig config) override { - config_ = config; + opts_.config_ = config; } void try_get_next_block(td::Timestamp timestamp, td::Promise promise); @@ -222,7 +222,7 @@ class FullNodeShardImpl : public FullNodeShard { } FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, + FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, @@ -269,7 +269,7 @@ class FullNodeShardImpl : public FullNodeShard { bool active_; - FullNodeConfig config_; + FullNodeOptions opts_; std::set my_ext_msg_broadcasts_; std::set processed_ext_msg_broadcasts_; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 658cb34e..e1951c36 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -139,7 +139,7 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise state, std void FullNodeImpl::update_shard_actor(ShardIdFull shard, bool active) { ShardInfo &info = shards_[shard]; if (info.actor.empty()) { - info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, adnl_, rldp_, + info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, client_, actor_id(this), active); if (!all_validators_.empty()) { td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); @@ -717,7 +717,7 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { nodes.push_back(p.second); } private_block_overlays_[key] = td::actor::create_actor( - "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_, + "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); update_validator_telemetry_collector(); } @@ -735,7 +735,7 @@ void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { old_actors.erase(it); } else { overlay.actors_[local_id] = td::actor::create_actor( - "CustomOverlay", local_id, params, zero_state_file_hash_, config_, keyring_, adnl_, rldp_, rldp2_, + "CustomOverlay", local_id, params, zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); } } @@ -794,7 +794,7 @@ void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const Block } FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - FullNodeConfig config, td::actor::ActorId keyring, + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId dht, td::actor::ActorId overlays, @@ -814,16 +814,16 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , client_(client) , db_root_(db_root) , started_promise_(std::move(started_promise)) - , config_(config) { + , opts_(opts) { } td::actor::ActorOwn FullNode::create( - ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, + ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, std::string db_root, td::Promise started_promise) { - return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, config, keyring, + return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, opts, keyring, adnl, rldp, rldp2, dht, overlays, validator_manager, client, db_root, std::move(started_promise)); } diff --git a/validator/full-node.h b/validator/full-node.h index fdb1bf3b..555082dc 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -55,6 +55,12 @@ struct FullNodeConfig { bool ext_messages_broadcast_disabled_ = false; }; +struct FullNodeOptions { + FullNodeConfig config_; + double public_broadcast_speed_multiplier_ = 1.0; + double private_broadcast_speed_multiplier_ = 1.0; +}; + struct CustomOverlayParams { std::string name_; std::vector nodes_; @@ -107,7 +113,7 @@ class FullNode : public td::actor::Actor { enum { broadcast_mode_public = 1, broadcast_mode_private_block = 2, broadcast_mode_custom = 4 }; static td::actor::ActorOwn create( - ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, + ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 0ea6fa0b..b4c79363 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -98,7 +98,7 @@ class FullNodeImpl : public FullNode { void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, @@ -141,7 +141,7 @@ class FullNodeImpl : public FullNode { std::set local_keys_; td::Promise started_promise_; - FullNodeConfig config_; + FullNodeOptions opts_; std::map> private_block_overlays_; bool broadcast_block_candidates_in_public_overlay_ = false; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 1817180d..110ccd81 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -373,6 +373,7 @@ void ValidatorGroup::create_session() { } CHECK(found); + config_.catchain_opts.broadcast_speed_multiplier = opts_->get_catchain_broadcast_speed_multiplier(); if (!config_.new_catchain_ids) { session_ = validatorsession::ValidatorSession::create(session_id_, config_, local_id_, std::move(vec), make_validator_session_callback(), keyring_, adnl_, rldp_, diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index e958d886..ace6b106 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -154,6 +154,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_fast_state_serializer_enabled() const override { return fast_state_serializer_enabled_; } + double get_catchain_broadcast_speed_multiplier() const override { + return catchain_broadcast_speed_multipliers_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -249,6 +252,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_fast_state_serializer_enabled(bool value) override { fast_state_serializer_enabled_ = value; } + void set_catchain_broadcast_speed_multiplier(double value) override { + catchain_broadcast_speed_multipliers_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -302,6 +308,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; bool fast_state_serializer_enabled_ = false; + double catchain_broadcast_speed_multipliers_; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 42b3f69a..5d6c0173 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -116,6 +116,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual bool get_state_serializer_enabled() const = 0; virtual td::Ref get_collator_options() const = 0; virtual bool get_fast_state_serializer_enabled() const = 0; + virtual double get_catchain_broadcast_speed_multiplier() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -148,6 +149,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_state_serializer_enabled(bool value) = 0; virtual void set_collator_options(td::Ref value) = 0; virtual void set_fast_state_serializer_enabled(bool value) = 0; + virtual void set_catchain_broadcast_speed_multiplier(double value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, From 61b9155d15d5a115f65fbb33bf095a7d53f96fba Mon Sep 17 00:00:00 2001 From: Sild Date: Fri, 21 Feb 2025 08:46:33 +0100 Subject: [PATCH 06/10] dont use instance after std::move (#1528) Co-authored-by: Dmitrii Korchagin --- tonlib/tonlib/TonlibClient.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index d917a57a..d73e715c 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -4619,6 +4619,8 @@ void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector std::vector> result_entries; result_entries.reserve(library_list.size()); std::vector not_cached_hashes; + not_cached_hashes.reserve(library_list.size()); + for (auto& library_hash : library_list) { if (libraries.key_exists(library_hash)) { auto library_content = vm::std_boc_serialize(libraries.lookup_ref(library_hash)).move_as_ok().as_slice().str(); @@ -4633,7 +4635,8 @@ void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector return; } - client_.send_query(ton::lite_api::liteServer_getLibrariesWithProof(ton::create_tl_lite_block_id(blkid), 1, std::move(not_cached_hashes)), + auto missed_lib_ids = not_cached_hashes; + client_.send_query(ton::lite_api::liteServer_getLibrariesWithProof(ton::create_tl_lite_block_id(blkid), 1, std::move(missed_lib_ids)), promise.wrap([self=this, blkid, result_entries = std::move(result_entries), not_cached_hashes] (td::Result> r_libraries) mutable -> td::Result> { From 1e8fdc05618bad116db34399ae691401e5cae1fb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 21 Feb 2025 10:52:23 +0300 Subject: [PATCH 07/10] Fix updateInit offset in storage (#1525) --- storage/PeerActor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/PeerActor.cpp b/storage/PeerActor.cpp index 48d45626..e140b4ce 100644 --- a/storage/PeerActor.cpp +++ b/storage/PeerActor.cpp @@ -251,7 +251,7 @@ void PeerActor::loop_update_init() { } s = s.substr(peer_init_offset_, UPDATE_INIT_BLOCK_SIZE); auto query = create_update_query(ton::create_tl_object( - td::BufferSlice(s), (int)peer_init_offset_, to_ton_api(node_state))); + td::BufferSlice(s), (int)peer_init_offset_ * 8, to_ton_api(node_state))); // take care about update_state_query initial state update_state_query_.state = node_state; @@ -502,11 +502,11 @@ void PeerActor::process_update_peer_parts(const tl_object_ptr(offset + i)); } From 1b70e483272c193926633e41eb357a700b0bc293 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Sun, 23 Feb 2025 13:01:33 +0100 Subject: [PATCH 08/10] Add option to build static tonlibjson and emulator (#1527) * add option to build static tonlibjson and emulator * do not export cmake project in case of static tonlibjson --- emulator/CMakeLists.txt | 8 +++----- tonlib/CMakeLists.txt | 8 +++++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index a0799541..663c8fd2 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -1,8 +1,6 @@ cmake_minimum_required(VERSION 3.5 FATAL_ERROR) -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() +option(EMULATOR_STATIC "Build emulator as static library" OFF) set(EMULATOR_STATIC_SOURCE transaction-emulator.cpp @@ -22,7 +20,7 @@ include(GenerateExportHeader) add_library(emulator_static STATIC ${EMULATOR_STATIC_SOURCE}) target_link_libraries(emulator_static PUBLIC ton_crypto smc-envelope) -if (USE_EMSCRIPTEN) +if (EMULATOR_STATIC OR USE_EMSCRIPTEN) add_library(emulator STATIC ${EMULATOR_SOURCE}) else() add_library(emulator SHARED ${EMULATOR_SOURCE}) @@ -35,7 +33,7 @@ else() endif() generate_export_header(emulator EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/emulator_export.h) -if (USE_EMSCRIPTEN) +if (EMULATOR_STATIC OR USE_EMSCRIPTEN) target_compile_definitions(emulator PUBLIC EMULATOR_STATIC_DEFINE) endif() target_include_directories(emulator PUBLIC diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index eb538361..3dbd628d 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.5 FATAL_ERROR) +option(TONLIBJSON_STATIC "Build tonlibjson as static library" OFF) + if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() @@ -90,7 +92,7 @@ set(TONLIB_JSON_HEADERS tonlib/tonlib_client_json.h) set(TONLIB_JSON_SOURCE tonlib/tonlib_client_json.cpp) include(GenerateExportHeader) -if (USE_EMSCRIPTEN) +if (TONLIBJSON_STATIC OR USE_EMSCRIPTEN) add_library(tonlibjson STATIC ${TONLIB_JSON_SOURCE}) else() add_library(tonlibjson SHARED ${TONLIB_JSON_SOURCE}) @@ -103,7 +105,7 @@ else() endif() generate_export_header(tonlibjson EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h) -if (USE_EMSCRIPTEN) +if (TONLIBJSON_STATIC OR USE_EMSCRIPTEN) target_compile_definitions(tonlibjson PUBLIC TONLIBJSON_STATIC_DEFINE) endif() target_include_directories(tonlibjson PUBLIC @@ -157,7 +159,7 @@ endif() install(FILES ${TONLIB_JSON_HEADERS} ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h DESTINATION include/tonlib/) -if (NOT USE_EMSCRIPTEN) +if (NOT USE_EMSCRIPTEN AND NOT TONLIBJSON_STATIC) install(EXPORT Tonlib FILE TonlibTargets.cmake NAMESPACE Tonlib:: From b3b2bd1c3c645a931a9aa6cbfc915c258c9012cc Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 27 Feb 2025 15:18:59 +0300 Subject: [PATCH 09/10] New extra currency behavior (#1539) --- common/global-version.h | 2 +- crypto/block/block.cpp | 29 ++++++ crypto/block/block.h | 2 + crypto/block/block.tlb | 2 +- crypto/block/mc-config.cpp | 1 + crypto/block/mc-config.h | 1 + crypto/block/transaction.cpp | 131 ++++++++++++++++++++++++---- crypto/block/transaction.h | 17 ++-- crypto/vm/boc.cpp | 8 +- crypto/vm/tonops.cpp | 12 ++- doc/GlobalVersions.md | 23 ++++- emulator/transaction-emulator.cpp | 11 ++- validator/impl/collator-impl.h | 14 ++- validator/impl/collator.cpp | 17 ++-- validator/impl/external-message.cpp | 16 ++-- validator/impl/validate-query.cpp | 6 +- validator/impl/validate-query.hpp | 1 + 17 files changed, 228 insertions(+), 65 deletions(-) diff --git a/common/global-version.h b/common/global-version.h index 533e5e8d..2308ce3e 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -const int SUPPORTED_VERSION = 9; +constexpr int SUPPORTED_VERSION = 10; } diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index 452d78a2..e0782240 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -1350,6 +1350,35 @@ bool CurrencyCollection::clamp(const CurrencyCollection& other) { return ok || invalidate(); } +bool CurrencyCollection::check_extra_currency_limit(td::uint32 max_currencies) const { + td::uint32 count = 0; + return vm::Dictionary{extra, 32}.check_for_each([&](td::Ref, td::ConstBitPtr, int) { + ++count; + return count <= max_currencies; + }); +} + +bool CurrencyCollection::remove_zero_extra_currencies(Ref& root, td::uint32 max_currencies) { + td::uint32 count = 0; + vm::Dictionary dict{root, 32}; + int res = dict.filter([&](const vm::CellSlice& cs, td::ConstBitPtr, int) -> int { + ++count; + if (count > max_currencies) { + return -1; + } + td::RefInt256 val = tlb::t_VarUInteger_32.as_integer(cs); + if (val.is_null()) { + return -1; + } + return val->sgn() > 0; + }); + if (res < 0) { + return false; + } + root = dict.get_root_cell(); + return true; +} + bool CurrencyCollection::operator==(const CurrencyCollection& other) const { return is_valid() && other.is_valid() && !td::cmp(grams, other.grams) && (extra.not_null() == other.extra.not_null()) && diff --git a/crypto/block/block.h b/crypto/block/block.h index f64f00a8..685005b4 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -391,6 +391,8 @@ struct CurrencyCollection { CurrencyCollection operator-(CurrencyCollection&& other) const; CurrencyCollection operator-(td::RefInt256 other_grams) const; bool clamp(const CurrencyCollection& other); + bool check_extra_currency_limit(td::uint32 max_currencies) const; + static bool remove_zero_extra_currencies(Ref& root, td::uint32 max_currencies); bool store(vm::CellBuilder& cb) const; bool store_or_zero(vm::CellBuilder& cb) const; bool fetch(vm::CellSlice& cs); diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index b8b40827..4a8bbc06 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -801,7 +801,7 @@ size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 - max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 = SizeLimitsConfig; + max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; // key is [ wc:int32 addr:uint256 ] diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 48a2d613..0f019b06 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -1960,6 +1960,7 @@ td::Result Config::do_get_size_limits_config(td::Refsgn() == 0 : ap.remaining_balance.is_zero()); ap.acc_status_change = ActionPhase::acst_deleted; - acc_status = Account::acc_deleted; + acc_status = (ap.remaining_balance.is_zero() ? Account::acc_deleted : Account::acc_uninit); was_deleted = true; } ap.success = true; @@ -2472,6 +2472,20 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, LOG(DEBUG) << "invalid destination address in a proposed outbound message"; return check_skip_invalid(36); // invalid destination address } + if (cfg.extra_currency_v2) { + CurrencyCollection value; + if (!value.unpack(info.value)) { + LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message"; + return check_skip_invalid(37); // invalid value:CurrencyCollection + } + if (!CurrencyCollection::remove_zero_extra_currencies(value.extra, cfg.size_limits.max_msg_extra_currencies)) { + LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message: too many currencies (max " + << cfg.size_limits.max_msg_extra_currencies << ")"; + // Dict should be valid, since it was checked in t_OutListNode.validate_ref, so error here means limit exceeded + return check_skip_invalid(41); // invalid value:CurrencyCollection : too many extra currencies + } + info.value = value.pack(); + } // fetch message pricing info const MsgPrices& msg_prices = cfg.fetch_msg_prices(to_mc || account.is_masterchain()); @@ -2524,7 +2538,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, }; add_used_storage(msg.init, 3); // message init add_used_storage(msg.body, 3); // message body (the root cell itself is not counted) - if (!ext_msg) { + if (!ext_msg && !cfg.extra_currency_v2) { add_used_storage(info.value->prefetch_ref(), 0); } auto collect_fine = [&] { @@ -2595,11 +2609,19 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (act_rec.mode & 0x80) { // attach all remaining balance to this message - req = ap.remaining_balance; + if (cfg.extra_currency_v2) { + req.grams = ap.remaining_balance.grams; + } else { + req = ap.remaining_balance; + } act_rec.mode &= ~1; // pay fees from attached value } else if (act_rec.mode & 0x40) { // attach all remaining balance of the inbound message (in addition to the original value) - req += msg_balance_remaining; + if (cfg.extra_currency_v2) { + req.grams += msg_balance_remaining.grams; + } else { + req += msg_balance_remaining; + } if (!(act_rec.mode & 1)) { req -= ap.action_fine; if (compute_phase) { @@ -2639,6 +2661,11 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, return check_skip_invalid(37); // not enough grams } + if (cfg.extra_currency_v2 && !req.check_extra_currency_limit(cfg.size_limits.max_msg_extra_currencies)) { + LOG(DEBUG) << "too many extra currencies in the message : max " << cfg.size_limits.max_msg_extra_currencies; + return check_skip_invalid(41); // to many extra currencies + } + Ref new_extra; if (!block::sub_extra_currency(ap.remaining_balance.extra, req.extra, new_extra)) { @@ -2680,7 +2707,11 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, // clear msg_balance_remaining if it has been used if (act_rec.mode & 0xc0) { - msg_balance_remaining.set_zero(); + if (cfg.extra_currency_v2) { + msg_balance_remaining.grams = td::zero_refint(); + } else { + msg_balance_remaining.set_zero(); + } } // update balance @@ -2754,8 +2785,13 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, ap.total_fwd_fees += fees_total; if ((act_rec.mode & 0xa0) == 0xa0) { - CHECK(ap.remaining_balance.is_zero()); - ap.acc_delete_req = ap.reserved_balance.is_zero(); + if (cfg.extra_currency_v2) { + CHECK(ap.remaining_balance.grams->sgn() == 0); + ap.acc_delete_req = ap.reserved_balance.grams->sgn() == 0; + } else { + CHECK(ap.remaining_balance.is_zero()); + ap.acc_delete_req = ap.reserved_balance.is_zero(); + } } ap.tot_msg_bits += sstat.bits + new_msg_bits; @@ -3026,7 +3062,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { bp.fwd_fees -= bp.fwd_fees_collected; total_fees += td::make_refint(bp.fwd_fees_collected); // serialize outbound message - info.created_lt = end_lt++; + info.created_lt = start_lt + 1 + out_msgs.size(); + end_lt++; info.created_at = now; vm::CellBuilder cb; CHECK(cb.store_long_bool(5, 4) // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool @@ -3107,6 +3144,7 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const { * Tries to update the storage statistics based on the old storage statistics and old account state without fully recomputing it. * * It succeeds if only root cell of AccountStorage is changed. + * old_cs and new_cell are AccountStorage without extra currencies (if global_version >= 10). * * @param old_stat The old storage statistics. * @param old_cs The old AccountStorage. @@ -3140,13 +3178,48 @@ static td::optional try_update_storage_stat(const vm::CellS return new_stat; } +/** + * Removes extra currencies dict from AccountStorage. + * + * This is used for computing account storage stats. + * + * @param storage_cs AccountStorage as CellSlice. + * + * @returns AccountStorage without extra currencies as Cell. + */ +static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { + block::gen::AccountStorage::Record rec; + if (!block::gen::csr_unpack(storage_cs, rec)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + if (rec.balance->size_refs() > 0) { + block::gen::CurrencyCollection::Record balance; + if (!block::gen::csr_unpack(rec.balance, balance)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + balance.other = vm::CellBuilder{}.store_zeroes(1).as_cellslice_ref(); + if (!block::gen::csr_pack(rec.balance, balance)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + } + td::Ref cell; + if (!block::gen::pack_cell(cell, rec)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + return cell; +} + namespace transaction { /** * Computes the new state of the account. * * @returns True if the state computation is successful, false otherwise. */ -bool Transaction::compute_state() { +bool Transaction::compute_state(const SerializeConfig& cfg) { if (new_total_state.not_null()) { return true; } @@ -3218,13 +3291,27 @@ bool Transaction::compute_state() { new_inner_state.clear(); } vm::CellStorageStat& stats = new_storage_stat; - auto new_stats = try_update_storage_stat(account.storage_stat, account.storage, storage); + td::Ref old_storage_for_stat = account.storage; + td::Ref new_storage_for_stat = storage; + if (cfg.extra_currency_v2) { + new_storage_for_stat = storage_without_extra_currencies(new_storage); + if (new_storage_for_stat.is_null()) { + return false; + } + if (old_storage_for_stat.not_null()) { + old_storage_for_stat = vm::load_cell_slice_ref(storage_without_extra_currencies(old_storage_for_stat)); + if (old_storage_for_stat.is_null()) { + return false; + } + } + } + auto new_stats = try_update_storage_stat(account.storage_stat, old_storage_for_stat, storage); if (new_stats) { stats = new_stats.unwrap(); } else { TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; - stats.add_used_storage(Ref(storage)).ensure(); + stats.add_used_storage(new_storage_for_stat).ensure(); if (timer.elapsed() > 0.1) { LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; } @@ -3260,11 +3347,11 @@ bool Transaction::compute_state() { * * @returns True if the serialization is successful, False otherwise. */ -bool Transaction::serialize() { +bool Transaction::serialize(const SerializeConfig& cfg) { if (root.not_null()) { return true; } - if (!compute_state()) { + if (!compute_state(cfg)) { return false; } vm::Dictionary dict{15}; @@ -3730,6 +3817,7 @@ bool Account::libraries_changed() const { * @param rand_seed Pointer to the random seed. Generates a new seed if the value is `td::Bits256::zero()`. * @param compute_phase_cfg Pointer to store the compute phase configuration. * @param action_phase_cfg Pointer to store the action phase configuration. + * @param serialize_cfg Pointer to store the serialize phase configuration. * @param masterchain_create_fee Pointer to store the masterchain create fee. * @param basechain_create_fee Pointer to store the basechain create fee. * @param wc The workchain ID. @@ -3738,15 +3826,15 @@ bool Account::libraries_changed() const { td::Status FetchConfigParams::fetch_config_params( const block::ConfigInfo& config, Ref* old_mparams, std::vector* storage_prices, StoragePhaseConfig* storage_phase_cfg, td::BitArray<256>* rand_seed, ComputePhaseConfig* compute_phase_cfg, - ActionPhaseConfig* action_phase_cfg, td::RefInt256* masterchain_create_fee, td::RefInt256* basechain_create_fee, - ton::WorkchainId wc, ton::UnixTime now) { + ActionPhaseConfig* action_phase_cfg, SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, ton::WorkchainId wc, ton::UnixTime now) { auto prev_blocks_info = config.get_prev_blocks_info(); if (prev_blocks_info.is_error()) { return prev_blocks_info.move_as_error_prefix( td::Status::Error(-668, "cannot fetch prev blocks info from masterchain configuration: ")); } return fetch_config_params(config, prev_blocks_info.move_as_ok(), old_mparams, storage_prices, storage_phase_cfg, - rand_seed, compute_phase_cfg, action_phase_cfg, masterchain_create_fee, + rand_seed, compute_phase_cfg, action_phase_cfg, serialize_cfg, masterchain_create_fee, basechain_create_fee, wc, now); } @@ -3761,6 +3849,7 @@ td::Status FetchConfigParams::fetch_config_params( * @param rand_seed Pointer to the random seed. Generates a new seed if the value is `td::Bits256::zero()`. * @param compute_phase_cfg Pointer to store the compute phase configuration. * @param action_phase_cfg Pointer to store the action phase configuration. + * @param serialize_cfg Pointer to store the serialize phase configuration. * @param masterchain_create_fee Pointer to store the masterchain create fee. * @param basechain_create_fee Pointer to store the basechain create fee. * @param wc The workchain ID. @@ -3770,8 +3859,8 @@ td::Status FetchConfigParams::fetch_config_params( const block::Config& config, td::Ref prev_blocks_info, Ref* old_mparams, std::vector* storage_prices, StoragePhaseConfig* storage_phase_cfg, td::BitArray<256>* rand_seed, ComputePhaseConfig* compute_phase_cfg, ActionPhaseConfig* action_phase_cfg, - td::RefInt256* masterchain_create_fee, td::RefInt256* basechain_create_fee, ton::WorkchainId wc, - ton::UnixTime now) { + SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, td::RefInt256* basechain_create_fee, + ton::WorkchainId wc, ton::UnixTime now) { *old_mparams = config.get_config_param(9); { auto res = config.get_storage_prices(); @@ -3843,6 +3932,10 @@ td::Status FetchConfigParams::fetch_config_params( action_phase_cfg->disable_custom_fess = config.get_global_version() >= 8; action_phase_cfg->reserve_extra_enabled = config.get_global_version() >= 9; action_phase_cfg->mc_blackhole_addr = config.get_burning_config().blackhole_addr; + action_phase_cfg->extra_currency_v2 = config.get_global_version() >= 10; + } + { + serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; } { // fetch block_grams_created diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 0f6952dc..8e612e6a 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -170,12 +170,17 @@ struct ActionPhaseConfig { bool message_skip_enabled{false}; bool disable_custom_fess{false}; bool reserve_extra_enabled{false}; + bool extra_currency_v2{false}; td::optional mc_blackhole_addr; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { return is_masterchain ? fwd_mc : fwd_std; } }; +struct SerializeConfig { + bool extra_currency_v2{false}; +}; + struct CreditPhase { td::RefInt256 due_fees_collected; block::CurrencyCollection credit; @@ -389,8 +394,8 @@ struct Transaction { bool prepare_action_phase(const ActionPhaseConfig& cfg); td::Status check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat = true); bool prepare_bounce_phase(const ActionPhaseConfig& cfg); - bool compute_state(); - bool serialize(); + bool compute_state(const SerializeConfig& cfg); + bool serialize(const SerializeConfig& cfg); td::uint64 gas_used() const { return compute_phase ? compute_phase->gas_used : 0; } @@ -428,14 +433,14 @@ struct FetchConfigParams { std::vector* storage_prices, StoragePhaseConfig* storage_phase_cfg, td::BitArray<256>* rand_seed, ComputePhaseConfig* compute_phase_cfg, ActionPhaseConfig* action_phase_cfg, - td::RefInt256* masterchain_create_fee, td::RefInt256* basechain_create_fee, - ton::WorkchainId wc, ton::UnixTime now); + SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, ton::WorkchainId wc, ton::UnixTime now); static td::Status fetch_config_params(const block::Config& config, Ref prev_blocks_info, Ref* old_mparams, std::vector* storage_prices, StoragePhaseConfig* storage_phase_cfg, td::BitArray<256>* rand_seed, ComputePhaseConfig* compute_phase_cfg, ActionPhaseConfig* action_phase_cfg, - td::RefInt256* masterchain_create_fee, td::RefInt256* basechain_create_fee, - ton::WorkchainId wc, ton::UnixTime now); + SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, ton::WorkchainId wc, ton::UnixTime now); }; } // namespace block diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 7ec8bdd1..72afb998 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1153,8 +1153,12 @@ td::Result CellStorageStat::add_used_storage(Refsecond; } } - vm::CellSlice cs{vm::NoVm{}, std::move(cell)}; - return add_used_storage(std::move(cs), kill_dup, skip_count_root); + vm::CellSlice cs{vm::NoVm{}, cell}; + TRY_RESULT(res, add_used_storage(std::move(cs), kill_dup, skip_count_root)); + if (kill_dup) { + seen[cell->get_hash()] = res; + } + return res; } void NewCellStorageStat::add_cell(Ref cell) { diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 5d90b8fd..aab1711f 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -1761,6 +1761,10 @@ int exec_send_message(VmState* st) { vm::VmStorageStat stat(max_cells); CellSlice cs = load_cell_slice(msg_cell); cs.skip_first(cs.size()); + if (st->get_global_version() >= 10 && have_extra_currencies) { + // Skip extra currency dict + cs.advance_refs(1); + } stat.add_storage(cs); if (!ext_msg) { @@ -1773,7 +1777,9 @@ int exec_send_message(VmState* st) { if (value.is_null()) { throw VmError{Excno::type_chk, "invalid param BALANCE"}; } - have_extra_currencies |= !tuple_index(balance, 1).as_cell().is_null(); + if (st->get_global_version() < 10) { + have_extra_currencies |= !tuple_index(balance, 1).as_cell().is_null(); + } } else if (mode & 64) { // value += value of incoming message Ref balance = get_param(st, 11).as_tuple(); if (balance.is_null()) { @@ -1784,7 +1790,9 @@ int exec_send_message(VmState* st) { throw VmError{Excno::type_chk, "invalid param INCOMINGVALUE"}; } value += balance_grams; - have_extra_currencies |= !tuple_index(balance, 1).as_cell().is_null(); + if (st->get_global_version() < 10) { + have_extra_currencies |= !tuple_index(balance, 1).as_cell().is_null(); + } } } diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index f4156ca0..77963e95 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -134,4 +134,25 @@ Example: if the last masterchain block seqno is `19071` then the list contains b - `PFXDICTADD`, `PFXDICTSET`, `PFXDICTREPLACE`, `PFXDICTDEL`, `GETGASFEE`, `GETSTORAGEFEE`, `GETFORWARDFEE`, `GETORIGINALFWDFEE`, `GETGASFEESIMPLE`, `GETFORWARDFEESIMPLE`, `HASHEXT` - Now setting the contract code to a library cell does not consume additional gas on execution of the code. - Temporary increase gas limit for some accounts (see [this post](https://t.me/tondev_news/129) for details, `override_gas_limit` in `transaction.cpp` for the list of accounts). -- Fix recursive jump to continuations with non-null control data. \ No newline at end of file +- Fix recursive jump to continuations with non-null control data. + +## Version 10 + +### Extra currencies +- Internal messages cannot carry more than 2 different extra currencies. The limit can be changed in size limits config (`ConfigParam 43`). +- Amount of an extra currency in an output action "send message" can be zero. + - In action phase zero values are automatically deleted from the dictionary before sending. + - However, the size of the extra currency dictionary in the "send message" action should not be greater than 2 (or the value in size limits config). +- Extra currency dictionary is not counted in message size and does not affect message fees. +- Message mode `+64` (carry all remaining message balance) is now considered as "carry all remaining TONs from message balance". +- Message mode `+128` (carry all remaining account balance) is now considered as "carry all remaining TONs from account balance". +- Message mode `+32` (delete account if balance is zero) deletes account if it has zero TONs, regardless of extra currencies. + - Deleted accounts with extra currencies become `account_uninit`, extra currencies remain on the account. +- `SENDMSG` in TVM calculates message size and fees without extra currencies, uses new `+64` and `+128` mode behavior. + - `SENDMSG` does not check the number of extra currencies. +- Extra currency dictionary is not counted in the account size and does not affect storage fees. + - Accounts with already existing extra currencies will get their sizes recomputed without EC only after modifying `AccountState`. + +### TVM changes +- `SENDMSG` calculates messages size and fees without extra currencies, uses new +64 and +128 mode behavior. + - `SENDMSG` does not check the number of extra currencies. diff --git a/emulator/transaction-emulator.cpp b/emulator/transaction-emulator.cpp index e87b2dfb..6267f9bd 100644 --- a/emulator/transaction-emulator.cpp +++ b/emulator/transaction-emulator.cpp @@ -16,6 +16,7 @@ td::Result> TransactionEmu block::StoragePhaseConfig storage_phase_cfg{&storage_prices}; block::ComputePhaseConfig compute_phase_cfg; block::ActionPhaseConfig action_phase_cfg; + block::SerializeConfig serialize_config; td::RefInt256 masterchain_create_fee, basechain_create_fee; if (!utime) { @@ -25,11 +26,9 @@ td::Result> TransactionEmu utime = (unsigned)std::time(nullptr); } - auto fetch_res = block::FetchConfigParams::fetch_config_params(*config_, prev_blocks_info_, &old_mparams, - &storage_prices, &storage_phase_cfg, - &rand_seed_, &compute_phase_cfg, - &action_phase_cfg, &masterchain_create_fee, - &basechain_create_fee, account.workchain, utime); + auto fetch_res = block::FetchConfigParams::fetch_config_params( + *config_, prev_blocks_info_, &old_mparams, &storage_prices, &storage_phase_cfg, &rand_seed_, &compute_phase_cfg, + &action_phase_cfg, &serialize_config, &masterchain_create_fee, &basechain_create_fee, account.workchain, utime); if(fetch_res.is_error()) { return fetch_res.move_as_error_prefix("cannot fetch config params "); } @@ -66,7 +65,7 @@ td::Result> TransactionEmu return std::make_unique(std::move(vm_log), vm_exit_code, elapsed); } - if (!trans->serialize()) { + if (!trans->serialize(serialize_config)) { return td::Status::Error(-669,"cannot serialize new transaction for smart contract "s + trans->account.addr.to_hex()); } diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index ce21bc5e..340e3a40 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -109,14 +109,11 @@ class Collator final : public td::actor::Actor { return 2; } - static td::Result> - impl_create_ordinary_transaction(Ref msg_root, - block::Account* acc, - UnixTime utime, LogicalTime lt, - block::StoragePhaseConfig* storage_phase_cfg, - block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg, - bool external, LogicalTime after_lt); + static td::Result> impl_create_ordinary_transaction( + Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, block::SerializeConfig* serialize_cfg, bool external, + LogicalTime after_lt); private: void start_up() override; @@ -177,6 +174,7 @@ class Collator final : public td::actor::Actor { block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; block::ComputePhaseConfig compute_phase_cfg_; block::ActionPhaseConfig action_phase_cfg_; + block::SerializeConfig serialize_cfg_; td::RefInt256 masterchain_create_fee_, basechain_create_fee_; std::unique_ptr block_limits_; std::unique_ptr block_limit_status_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index d5c41853..2a6d7a2b 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1995,12 +1995,9 @@ bool Collator::init_lt() { * @returns True if the configuration parameters were successfully fetched and initialized, false otherwise. */ bool Collator::fetch_config_params() { - auto res = block::FetchConfigParams::fetch_config_params(*config_, - &old_mparams_, &storage_prices_, &storage_phase_cfg_, - &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, - &masterchain_create_fee_, &basechain_create_fee_, - workchain(), now_ - ); + auto res = block::FetchConfigParams::fetch_config_params( + *config_, &old_mparams_, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &serialize_cfg_, &masterchain_create_fee_, &basechain_create_fee_, workchain(), now_); if (res.is_error()) { return fatal_error(res.move_as_error()); } @@ -2750,7 +2747,7 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t return fatal_error(td::Status::Error( -666, std::string{"cannot create action phase of a new transaction for smart contract "} + smc_addr.to_hex())); } - if (!trans->serialize()) { + if (!trans->serialize(serialize_cfg_)) { return fatal_error(td::Status::Error( -666, std::string{"cannot serialize new transaction for smart contract "} + smc_addr.to_hex())); } @@ -2834,7 +2831,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, after_lt = std::max(after_lt, it->second); } auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, external, after_lt); + &action_phase_cfg_, &serialize_cfg_, external, after_lt); if (res.is_error()) { auto error = res.move_as_error(); if (error.code() == -701) { @@ -2885,6 +2882,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, * @param storage_phase_cfg The configuration for the storage phase of the transaction. * @param compute_phase_cfg The configuration for the compute phase of the transaction. * @param action_phase_cfg The configuration for the action phase of the transaction. + * @param serialize_cfg The configuration for the serialization of the transaction. * @param external Flag indicating if the message is external. * @param after_lt The logical time after which the transaction should occur. Used only for external messages. * @@ -2898,6 +2896,7 @@ td::Result> Collator::impl_crea block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, block::ActionPhaseConfig* action_phase_cfg, + block::SerializeConfig* serialize_cfg, bool external, LogicalTime after_lt) { if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain @@ -2965,7 +2964,7 @@ td::Result> Collator::impl_crea return td::Status::Error( -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - if (!trans->serialize()) { + if (!trans->serialize(*serialize_cfg)) { return td::Status::Error(-669, "cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); } return std::move(trans); diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 2fdb491b..8b1f5eb7 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -136,13 +136,12 @@ td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc, td::BitArray<256> rand_seed_; block::ComputePhaseConfig compute_phase_cfg_; block::ActionPhaseConfig action_phase_cfg_; + block::SerializeConfig serialize_config_; td::RefInt256 masterchain_create_fee, basechain_create_fee; - auto fetch_res = block::FetchConfigParams::fetch_config_params(*config, &old_mparams, - &storage_prices_, &storage_phase_cfg_, - &rand_seed_, &compute_phase_cfg_, - &action_phase_cfg_, &masterchain_create_fee, - &basechain_create_fee, wc, utime); + auto fetch_res = block::FetchConfigParams::fetch_config_params( + *config, &old_mparams, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &serialize_config_, &masterchain_create_fee, &basechain_create_fee, wc, utime); if(fetch_res.is_error()) { auto error = fetch_res.move_as_error(); LOG(DEBUG) << "Cannot fetch config params: " << error.message(); @@ -152,10 +151,9 @@ td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc, compute_phase_cfg_.with_vm_log = true; compute_phase_cfg_.stop_on_accept_message = true; - auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, - &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, - true, lt); + auto res = + Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, &serialize_config_, true, lt); if(res.is_error()) { auto error = res.move_as_error(); LOG(DEBUG) << "Cannot run message on account: " << error.message(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 31c30e90..90966d82 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1004,6 +1004,10 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.disable_custom_fess = config_->get_global_version() >= 8; action_phase_cfg_.reserve_extra_enabled = config_->get_global_version() >= 9; action_phase_cfg_.mc_blackhole_addr = config_->get_burning_config().blackhole_addr; + action_phase_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + } + { + serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; } { // fetch block_grams_created @@ -5608,7 +5612,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT return reject_query(PSTRING() << "cannot re-create bounce phase of transaction " << lt << " for smart contract " << addr.to_hex()); } - if (!trs->serialize()) { + if (!trs->serialize(serialize_cfg_)) { return reject_query(PSTRING() << "cannot re-create the serialization of transaction " << lt << " for smart contract " << addr.to_hex()); } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 90c368ff..60f0cc8a 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -205,6 +205,7 @@ class ValidateQuery : public td::actor::Actor { block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; block::ComputePhaseConfig compute_phase_cfg_; block::ActionPhaseConfig action_phase_cfg_; + block::SerializeConfig serialize_cfg_; td::RefInt256 masterchain_create_fee_, basechain_create_fee_; std::vector neighbors_; From 44e7e091b28a15b362def30ff62602158e28b7f0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 27 Feb 2025 15:41:21 +0300 Subject: [PATCH 10/10] Use HashSet/HashMap in storage stat (#1540) --- crypto/vm/boc.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 8adf240f..17e7eb69 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -101,9 +101,9 @@ class NewCellStorageStat { private: const CellUsageTree* usage_tree_; - std::set seen_; + td::HashSet seen_; Stat stat_; - std::set proof_seen_; + td::HashSet proof_seen_; Stat proof_stat_; const NewCellStorageStat* parent_{nullptr}; @@ -117,7 +117,7 @@ struct CellStorageStat { struct CellInfo { td::uint32 max_merkle_depth = 0; }; - std::map seen; + td::HashMap seen; CellStorageStat() : cells(0), bits(0), public_cells(0) { } explicit CellStorageStat(unsigned long long limit_cells) @@ -173,7 +173,7 @@ class ProofStorageStat { enum CellStatus { c_none = 0, c_prunned = 1, c_loaded = 2 }; - std::map cells_; + td::HashMap cells_; td::uint64 proof_size_ = 0; };