external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Merge branch 'testnet' into accelerator

Browse source
This commit is contained in:
SpyCheese 2025-02-21 12:19:27 +03:00
commit bf884b5611
55 changed files with 866 additions and 326 deletions
Download patch file Download diff file Expand all files Collapse all files

6
catchain/catchain-receiver.cpp
View file

@ -526,10 +526,12 @@ void CatChainReceiverImpl::start_up() {
for (td::uint32 i = 0; i < get_sources_cnt(); i++) {
root_keys.emplace(get_source(i)->get_hash(), OVERLAY_MAX_ALLOWED_PACKET_SIZE);
}
td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay,
overlay::OverlayOptions overlay_options;
overlay_options.broadcast_speed_multiplier_ = opts_.broadcast_speed_multiplier;
td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay_ex,
get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids),
make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)},
R"({ "type": "catchain" })");
R"({ "type": "catchain" })", std::move(overlay_options));
CHECK(root_block_);

7
crypto/block/block.cpp
View file

@ -360,7 +360,6 @@ MsgProcessedUptoCollection::MsgProcessedUptoCollection(ton::ShardIdFull _owner,
z.shard = key.get_uint(64);
z.mc_seqno = (unsigned)((key + 64).get_uint(32));
z.last_inmsg_lt = value.write().fetch_ulong(64);
// std::cerr << "ProcessedUpto shard " << std::hex << z.shard << std::dec << std::endl;
return value.write().fetch_bits_to(z.last_inmsg_hash) && z.shard && ton::shard_contains(owner.shard, z.shard);
});
}
@ -886,8 +885,10 @@ td::Status ShardState::unpack_out_msg_queue_info(Ref<vm::Cell> out_msg_queue_inf
out_msg_queue_ =
std::make_unique<vm::AugmentedDictionary>(std::move(qinfo.out_queue), 352, block::tlb::aug_OutMsgQueue);
if (verbosity >= 3 * 1) {
LOG(DEBUG) << "unpacking ProcessedUpto of our previous block " << id_.to_str();
block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info);
FLOG(DEBUG) {
sb << "unpacking ProcessedUpto of our previous block " << id_.to_str();
block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info);
};
}
if (!block::gen::t_ProcessedInfo.validate_csr(1024, qinfo.proc_info)) {
return td::Status::Error(

14
crypto/block/mc-config.cpp
View file

@ -163,8 +163,11 @@ td::Status ConfigInfo::unpack() {
}
gen::McStateExtra::Record extra_info;
if (!tlb::unpack_cell(state_extra_root_, extra_info)) {
vm::load_cell_slice(state_extra_root_).print_rec(std::cerr);
block::gen::t_McStateExtra.print_ref(std::cerr, state_extra_root_);
FLOG(WARNING) {
sb << "state extra information is invalid: ";
vm::load_cell_slice(state_extra_root_).print_rec(sb);
block::gen::t_McStateExtra.print_ref(sb, state_extra_root_);
};
return td::Status::Error("state extra information is invalid");
}
gen::ValidatorInfo::Record validator_info;
@ -1067,7 +1070,6 @@ Ref<McShardHash> ShardConfig::get_shard_hash(ton::ShardIdFull id, bool exact) co
ton::ShardIdFull true_id;
vm::CellSlice cs;
if (get_shard_hash_raw(cs, id, true_id, exact)) {
// block::gen::t_ShardDescr.print(std::cerr, vm::CellSlice{cs});
return McShardHash::unpack(cs, true_id);
} else {
return {};
@ -1637,8 +1639,10 @@ bool ShardConfig::set_shard_info(ton::ShardIdFull shard, Ref<vm::Cell> value) {
if (!gen::t_BinTree_ShardDescr.validate_ref(1024, value)) {
LOG(ERROR) << "attempting to store an invalid (BinTree ShardDescr) at shard configuration position "
<< shard.to_str();
gen::t_BinTree_ShardDescr.print_ref(std::cerr, value);
vm::load_cell_slice(value).print_rec(std::cerr);
FLOG(WARNING) {
gen::t_BinTree_ShardDescr.print_ref(sb, value);
vm::load_cell_slice(value).print_rec(sb);
};
return false;
}
auto root = shard_hashes_dict_->lookup_ref(td::BitArray<32>{shard.workchain});

1
crypto/block/output-queue-merger.cpp
View file

@ -138,7 +138,6 @@ void OutputQueueMerger::add_root(int src, Ref<vm::Cell> outmsg_root, td::int32 m
if (outmsg_root.is_null()) {
return;
}
//block::gen::HashmapAug{352, block::gen::t_EnqueuedMsg, block::gen::t_uint64}.print_ref(std::cerr, outmsg_root);
auto kv = std::make_unique<MsgKeyValue>(src, std::move(outmsg_root));
if (kv->replace_by_prefix(common_pfx.cbits(), common_pfx_len)) {
heap.push_back(std::move(kv));

95
crypto/block/transaction.cpp
View file

@ -446,8 +446,10 @@ bool Account::unpack(Ref<vm::CellSlice> shard_account, ton::UnixTime now, bool s
return false;
}
if (verbosity > 2) {
shard_account->print_rec(std::cerr, 2);
block::gen::t_ShardAccount.print(std::cerr, *shard_account);
FLOG(INFO) {
shard_account->print_rec(sb, 2);
block::gen::t_ShardAccount.print(sb, shard_account);
};
}
block::gen::ShardAccount::Record acc_info;
if (!(block::tlb::t_ShardAccount.validate_csr(shard_account) && tlb::unpack_exact(shard_account.write(), acc_info))) {
@ -737,9 +739,11 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig*
return false;
}
if (verbosity > 2) {
fprintf(stderr, "unpacking inbound message for a new transaction: ");
block::gen::t_Message_Any.print_ref(std::cerr, in_msg);
load_cell_slice(in_msg).print_rec(std::cerr);
FLOG(INFO) {
sb << "unpacking inbound message for a new transaction: ";
block::gen::t_Message_Any.print_ref(sb, in_msg);
load_cell_slice(in_msg).print_rec(sb);
};
}
auto cs = vm::load_cell_slice(in_msg);
int tag = block::gen::t_CommonMsgInfo.get_tag(cs);
@ -1550,11 +1554,13 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom
cp.actions = impl.get_c5();
int out_act_num = output_actions_count(cp.actions);
if (verbosity > 2) {
std::cerr << "new smart contract data: ";
bool can_be_special = true;
load_cell_slice_special(cp.new_data, can_be_special).print_rec(std::cerr);
std::cerr << "output actions: ";
block::gen::OutList{out_act_num}.print_ref(std::cerr, cp.actions);
FLOG(INFO) {
sb << "new smart contract data: ";
bool can_be_special = true;
load_cell_slice_special(cp.new_data, can_be_special).print_rec(sb);
sb << "output actions: ";
block::gen::OutList{out_act_num}.print_ref(sb, cp.actions);
};
}
}
cp.mode = 0;
@ -1619,7 +1625,6 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
if (in_msg_state.not_null()) {
LOG(DEBUG) << "HASH(in_msg_state) = " << in_msg_state->get_hash().bits().to_hex(256)
<< ", account_state_hash = " << account.state_hash.to_hex();
// vm::load_cell_slice(in_msg_state).print_rec(std::cerr);
} else {
LOG(DEBUG) << "in_msg_state is null";
}
@ -1775,11 +1780,13 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
cp.actions = vm.get_committed_state().c5; // c5 -> action list
int out_act_num = output_actions_count(cp.actions);
if (verbosity > 2) {
std::cerr << "new smart contract data: ";
bool can_be_special = true;
load_cell_slice_special(cp.new_data, can_be_special).print_rec(std::cerr);
std::cerr << "output actions: ";
block::gen::OutList{out_act_num}.print_ref(std::cerr, cp.actions);
FLOG(INFO) {
sb << "new smart contract data: ";
bool can_be_special = true;
load_cell_slice_special(cp.new_data, can_be_special).print_rec(sb);
sb << "output actions: ";
block::gen::OutList{out_act_num}.print_ref(sb, cp.actions);
};
}
}
cp.mode = 0;
@ -2725,14 +2732,18 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap,
}
if (!block::gen::t_Message_Any.validate_ref(new_msg)) {
LOG(ERROR) << "generated outbound message is not a valid (Message Any) according to automated check";
block::gen::t_Message_Any.print_ref(std::cerr, new_msg);
vm::load_cell_slice(new_msg).print_rec(std::cerr);
FLOG(INFO) {
block::gen::t_Message_Any.print_ref(sb, new_msg);
vm::load_cell_slice(new_msg).print_rec(sb);
};
collect_fine();
return -1;
}
if (verbosity > 2) {
std::cerr << "converted outbound message: ";
block::gen::t_Message_Any.print_ref(std::cerr, new_msg);
FLOG(INFO) {
sb << "converted outbound message: ";
block::gen::t_Message_Any.print_ref(sb, new_msg);
};
}
ap.msgs_created++;
@ -3045,8 +3056,10 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) {
}
CHECK(cb.finalize_to(bp.out_msg));
if (verbosity > 2) {
LOG(INFO) << "generated bounced message: ";
block::gen::t_Message_Any.print_ref(std::cerr, bp.out_msg);
FLOG(INFO) {
sb << "generated bounced message: ";
block::gen::t_Message_Any.print_ref(sb, bp.out_msg);
};
}
out_msgs.push_back(bp.out_msg);
bp.ok = true;
@ -3167,11 +3180,13 @@ bool Transaction::compute_state() {
auto frozen_state = cb2.finalize();
frozen_hash = frozen_state->get_hash().bits();
if (verbosity >= 3 * 1) { // !!!DEBUG!!!
std::cerr << "freezing state of smart contract: ";
block::gen::t_StateInit.print_ref(std::cerr, frozen_state);
CHECK(block::gen::t_StateInit.validate_ref(frozen_state));
CHECK(block::tlb::t_StateInit.validate_ref(frozen_state));
std::cerr << "with hash " << frozen_hash.to_hex() << std::endl;
FLOG(INFO) {
sb << "freezing state of smart contract: ";
block::gen::t_StateInit.print_ref(sb, frozen_state);
CHECK(block::gen::t_StateInit.validate_ref(frozen_state));
CHECK(block::tlb::t_StateInit.validate_ref(frozen_state));
sb << "with hash " << frozen_hash.to_hex();
};
}
}
new_code.clear();
@ -3229,8 +3244,10 @@ bool Transaction::compute_state() {
CHECK(cb.append_data_cell_bool(std::move(storage)));
new_total_state = cb.finalize();
if (verbosity > 2) {
std::cerr << "new account state: ";
block::gen::t_Account.print_ref(std::cerr, new_total_state);
FLOG(INFO) {
sb << "new account state: ";
block::gen::t_Account.print_ref(sb, new_total_state);
};
}
CHECK(block::tlb::t_Account.validate_ref(new_total_state));
return true;
@ -3322,22 +3339,28 @@ bool Transaction::serialize() {
return false;
}
if (verbosity >= 3 * 1) {
std::cerr << "new transaction: ";
block::gen::t_Transaction.print_ref(std::cerr, root);
vm::load_cell_slice(root).print_rec(std::cerr);
FLOG(INFO) {
sb << "new transaction: ";
block::gen::t_Transaction.print_ref(sb, root);
vm::load_cell_slice(root).print_rec(sb);
};
}
if (!block::gen::t_Transaction.validate_ref(4096, root)) {
LOG(ERROR) << "newly-generated transaction failed to pass automated validation:";
vm::load_cell_slice(root).print_rec(std::cerr);
block::gen::t_Transaction.print_ref(std::cerr, root);
FLOG(INFO) {
vm::load_cell_slice(root).print_rec(sb);
block::gen::t_Transaction.print_ref(sb, root);
};
root.clear();
return false;
}
if (!block::tlb::t_Transaction.validate_ref(4096, root)) {
LOG(ERROR) << "newly-generated transaction failed to pass hand-written validation:";
vm::load_cell_slice(root).print_rec(std::cerr);
block::gen::t_Transaction.print_ref(std::cerr, root);
FLOG(INFO) {
vm::load_cell_slice(root).print_rec(sb);
block::gen::t_Transaction.print_ref(sb, root);
};
root.clear();
return false;
}

7
crypto/tl/tlblib.cpp
View file

@ -196,6 +196,13 @@ bool TLB::print_ref(std::ostream& os, Ref<vm::Cell> cell_ref, int indent, int re
return pp.fail_unless(print_ref(pp, std::move(cell_ref)));
}
bool TLB::print_ref(td::StringBuilder& sb, Ref<vm::Cell> cell_ref, int indent, int rec_limit) const {
std::ostringstream ss;
auto result = print_ref(ss, std::move(cell_ref), indent, rec_limit);
sb << ss.str();
return result;
}
std::string TLB::as_string_skip(vm::CellSlice& cs, int indent) const {
std::ostringstream os;
print_skip(os, cs, indent);

7
crypto/tl/tlblib.hpp
View file

@ -246,7 +246,14 @@ class TLB {
bool print(std::ostream& os, Ref<vm::CellSlice> cs_ref, int indent = 0, int rec_limit = 0) const {
return print(os, *cs_ref, indent, rec_limit);
}
bool print(td::StringBuilder& sb, Ref<vm::CellSlice> cs_ref, int indent = 0, int rec_limit = 0) const {
std::ostringstream ss;
auto result = print(ss, *cs_ref, indent, rec_limit);
sb << ss.str();
return result;
}
bool print_ref(std::ostream& os, Ref<vm::Cell> cell_ref, int indent = 0, int rec_limit = 0) const;
bool print_ref(td::StringBuilder& sb, Ref<vm::Cell> cell_ref, int indent = 0, int rec_limit = 0) const;
bool print_ref(int rec_limit, std::ostream& os, Ref<vm::Cell> cell_ref, int indent = 0) const {
return print_ref(os, std::move(cell_ref), indent, rec_limit);
}

7
crypto/vm/cells/CellSlice.cpp
View file

@ -1026,6 +1026,13 @@ bool CellSlice::print_rec(std::ostream& os, int indent) const {
return print_rec(os, &limit, indent);
}
bool CellSlice::print_rec(td::StringBuilder& sb, int indent) const {
std::ostringstream ss;
auto result = print_rec(ss, indent);
sb << ss.str();
return result;
}
bool CellSlice::print_rec(int limit, std::ostream& os, int indent) const {
return print_rec(os, &limit, indent);
}

1
crypto/vm/cells/CellSlice.h
View file

@ -257,6 +257,7 @@ class CellSlice : public td::CntObject {
void dump(std::ostream& os, int level = 0, bool endl = true) const;
void dump_hex(std::ostream& os, int mode = 0, bool endl = false) const;
bool print_rec(std::ostream& os, int indent = 0) const;
bool print_rec(td::StringBuilder& sb, int indent = 0) const;
bool print_rec(std::ostream& os, int* limit, int indent = 0) const;
bool print_rec(int limit, std::ostream& os, int indent = 0) const;
void error() const {

12
overlay/overlay-fec.cpp
View file

@ -32,7 +32,7 @@ void OverlayOutboundFecBroadcast::alarm() {
fec_type_.size(), flags_, std::move(X.data), X.id, fec_type_, date_);
}
alarm_timestamp() = td::Timestamp::in(0.010);
alarm_timestamp() = td::Timestamp::in(delay_);
if (seqno_ >= to_send_) {
stop();
@ -46,8 +46,9 @@ void OverlayOutboundFecBroadcast::start_up() {
OverlayOutboundFecBroadcast::OverlayOutboundFecBroadcast(td::BufferSlice data, td::uint32 flags,
td::actor::ActorId<OverlayImpl> overlay,
PublicKeyHash local_id)
PublicKeyHash local_id, double speed_multiplier)
: flags_(flags) {
delay_ /= speed_multiplier;
CHECK(data.size() <= (1 << 27));
local_id_ = local_id;
overlay_ = std::move(overlay);
@ -63,9 +64,10 @@ OverlayOutboundFecBroadcast::OverlayOutboundFecBroadcast(td::BufferSlice data, t
}
td::actor::ActorId<OverlayOutboundFecBroadcast> OverlayOutboundFecBroadcast::create(
td::BufferSlice data, td::uint32 flags, td::actor::ActorId<OverlayImpl> overlay, PublicKeyHash local_id) {
return td::actor::create_actor<OverlayOutboundFecBroadcast>(td::actor::ActorOptions().with_name("bcast"),
std::move(data), flags, overlay, local_id)
td::BufferSlice data, td::uint32 flags, td::actor::ActorId<OverlayImpl> overlay, PublicKeyHash local_id,
double speed_multiplier) {
return td::actor::create_actor<OverlayOutboundFecBroadcast>(
td::actor::ActorOptions().with_name("bcast"), std::move(data), flags, overlay, local_id, speed_multiplier)
.release();
}

5
overlay/overlay-fec.hpp
View file

@ -37,6 +37,7 @@ class OverlayOutboundFecBroadcast : public td::actor::Actor {
PublicKeyHash local_id_;
Overlay::BroadcastDataHash data_hash_;
td::uint32 flags_ = 0;
double delay_ = 0.010;
td::int32 date_;
std::unique_ptr<td::fec::Encoder> encoder_;
td::actor::ActorId<OverlayImpl> overlay_;
@ -45,9 +46,9 @@ class OverlayOutboundFecBroadcast : public td::actor::Actor {
public:
static td::actor::ActorId<OverlayOutboundFecBroadcast> create(td::BufferSlice data, td::uint32 flags,
td::actor::ActorId<OverlayImpl> overlay,
PublicKeyHash local_id);
PublicKeyHash local_id, double speed_multiplier = 1.0);
OverlayOutboundFecBroadcast(td::BufferSlice data, td::uint32 flags, td::actor::ActorId<OverlayImpl> overlay,
PublicKeyHash local_id);
PublicKeyHash local_id, double speed_multiplier = 1.0);
void alarm() override;
void start_up() override;

6
overlay/overlay.cpp
View file

@ -63,7 +63,7 @@ td::actor::ActorOwn<Overlay> Overlay::create_private(
return td::actor::create_actor<OverlayImpl>(
overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node, local_id, std::move(overlay_id),
OverlayType::FixedMemberList, std::move(nodes), std::vector<PublicKeyHash>(), OverlayMemberCertificate{},
std::move(callback), std::move(rules), std::move(scope));
std::move(callback), std::move(rules), std::move(scope), std::move(opts));
}
td::actor::ActorOwn<Overlay> Overlay::create_semiprivate(
@ -99,6 +99,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor
overlay_id_ = id_full_.compute_short_id();
frequent_dht_lookup_ = opts_.frequent_dht_lookup_;
peer_list_.local_member_flags_ = opts_.local_overlay_member_flags_;
opts_.broadcast_speed_multiplier_ = std::max(opts_.broadcast_speed_multiplier_, 1e-9);
VLOG(OVERLAY_INFO) << this << ": creating";
@ -490,7 +491,8 @@ void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td
VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid";
return;
}
OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as);
OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as,
opts_.broadcast_speed_multiplier_);
}
void OverlayImpl::print(td::StringBuilder &sb) {

1
overlay/overlays.h
View file

@ -269,6 +269,7 @@ struct OverlayOptions {
td::uint32 nodes_to_send_ = 4;
td::uint32 propagate_broadcast_to_ = 5;
td::uint32 default_permanent_members_flags_ = 0;
double broadcast_speed_multiplier_ = 1.0;
};
class Overlays : public td::actor::Actor {

8
storage/PeerActor.cpp
View file

@ -251,7 +251,7 @@ void PeerActor::loop_update_init() {
}
s = s.substr(peer_init_offset_, UPDATE_INIT_BLOCK_SIZE);
auto query = create_update_query(ton::create_tl_object<ton::ton_api::storage_updateInit>(
td::BufferSlice(s), (int)peer_init_offset_, to_ton_api(node_state)));
td::BufferSlice(s), (int)peer_init_offset_ * 8, to_ton_api(node_state)));
// take care about update_state_query initial state
update_state_query_.state = node_state;
@ -502,11 +502,11 @@ void PeerActor::process_update_peer_parts(const tl_object_ptr<ton_api::storage_U
},
[&](const ton::ton_api::storage_updateState &state) {},
[&](const ton::ton_api::storage_updateInit &init) {
LOG(DEBUG) << "Processing updateInit query (offset=" << init.have_pieces_offset_ * 8 << ")";
LOG(DEBUG) << "Processing updateInit query (offset=" << init.have_pieces_offset_ << ")";
td::Bitset new_bitset;
new_bitset.set_raw(init.have_pieces_.as_slice().str());
size_t offset = init.have_pieces_offset_ * 8;
for (auto size = new_bitset.size(), i = size_t(0); i < size; i++) {
size_t offset = init.have_pieces_offset_;
for (auto size = new_bitset.size(), i = (size_t)0; i < size; i++) {
if (new_bitset.get(i)) {
add_piece(static_cast<PartId>(offset + i));
}

4
tdutils/td/utils/Time.h
View file

@ -128,6 +128,10 @@ inline Timestamp &operator+=(Timestamp &a, double b) {
return a;
}
inline double operator-(const Timestamp &a, const Timestamp &b) {
return a.at() - b.at();
}
template <class StorerT>
void store(const Timestamp &timestamp, StorerT &storer) {
storer.store_binary(timestamp.at() - Time::now() + Clocks::system());

4
tdutils/td/utils/logging.h
View file

@ -264,8 +264,8 @@ class Logger {
sb_ << other;
return *this;
}
LambdaPrintHelper<td::Logger> operator<<(const LambdaPrint &) {
return LambdaPrintHelper<td::Logger>{*this};
LambdaPrintHelper<td::StringBuilder> operator<<(const LambdaPrint &) {
return LambdaPrintHelper<td::StringBuilder>{sb_};
}
MutableCSlice as_cslice() {

1
ton/ton-types.h
View file

@ -547,6 +547,7 @@ struct CatChainOptions {
td::uint64 max_block_height_coeff = 0;
bool debug_disable_db = false;
double broadcast_speed_multiplier = 1.0;
};
struct ValidatorSessionConfig {

5
tonlib/tonlib/TonlibClient.cpp
View file

@ -4745,6 +4745,8 @@ void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector<td::Bits256>
std::vector<object_ptr<tonlib_api::smc_libraryEntry>> result_entries;
result_entries.reserve(library_list.size());
std::vector<td::Bits256> not_cached_hashes;
not_cached_hashes.reserve(library_list.size());
for (auto& library_hash : library_list) {
if (libraries.key_exists(library_hash)) {
auto library_content = vm::std_boc_serialize(libraries.lookup_ref(library_hash)).move_as_ok().as_slice().str();
@ -4759,7 +4761,8 @@ void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector<td::Bits256>
return;
}
client_.send_query(ton::lite_api::liteServer_getLibrariesWithProof(ton::create_tl_lite_block_id(blkid), 1, std::move(not_cached_hashes)),
auto missed_lib_ids = not_cached_hashes;
client_.send_query(ton::lite_api::liteServer_getLibrariesWithProof(ton::create_tl_lite_block_id(blkid), 1, std::move(missed_lib_ids)),
promise.wrap([self=this, blkid, result_entries = std::move(result_entries), not_cached_hashes]
(td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResultWithProof>> r_libraries) mutable
-> td::Result<tonlib_api::object_ptr<tonlib_api::smc_libraryResult>> {

55
validator-engine/validator-engine.cpp
View file

@ -1637,6 +1637,7 @@ td::Status ValidatorEngine::load_global_config() {
}
validator_options_.write().set_hardforks(std::move(h));
validator_options_.write().set_fast_state_serializer_enabled(fast_state_serializer_enabled_);
validator_options_.write().set_catchain_broadcast_speed_multiplier(broadcast_speed_multiplier_catchain_);
for (auto& id : config_.collator_node_whitelist) {
validator_options_.write().set_collator_node_whitelisted_validator(id, true);
@ -2130,7 +2131,8 @@ void ValidatorEngine::started_overlays() {
void ValidatorEngine::start_validator() {
validator_options_.write().set_allow_blockchain_init(config_.validators.size() > 0);
validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled);
validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled &&
!state_serializer_disabled_flag_);
load_collator_options();
validator_manager_ = ton::validator::ValidatorManagerFactory::create(
@ -2177,9 +2179,13 @@ void ValidatorEngine::start_full_node() {
R.ensure();
td::actor::send_closure(SelfId, &ValidatorEngine::started_full_node);
});
ton::validator::fullnode::FullNodeOptions full_node_options{
.config_ = config_.full_node_config,
.public_broadcast_speed_multiplier_ = broadcast_speed_multiplier_public_,
.private_broadcast_speed_multiplier_ = broadcast_speed_multiplier_private_};
full_node_ = ton::validator::fullnode::FullNode::create(
short_id, full_node_id_, validator_options_->zero_block_id().file_hash, config_.full_node_config,
keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(),
short_id, full_node_id_, validator_options_->zero_block_id().file_hash,
full_node_options, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(),
default_dht_node_.is_zero() ? td::actor::ActorId<ton::dht::Dht>{} : dht_nodes_[default_dht_node_].get(),
overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P));
for (auto &v : config_.validators) {
@ -4398,7 +4404,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateS
promise.set_value(ton::create_serialize_tl_object<ton::ton_api::engine_validator_success>());
return;
}
validator_options_.write().set_state_serializer_enabled(query.enabled_);
validator_options_.write().set_state_serializer_enabled(query.enabled_ && !state_serializer_disabled_flag_);
td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options,
validator_options_);
config_.state_serializer_enabled = query.enabled_;
@ -5395,6 +5401,47 @@ int main(int argc, char *argv[]) {
td::actor::send_closure(x, &ValidatorEngine::set_validator_telemetry_filename, s);
});
});
p.add_option(
'\0', "disable-state-serializer",
"disable persistent state serializer (similar to set-state-serializer-enabled 0 in validator console)", [&]() {
acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_state_serializer_disabled_flag); });
});
p.add_checked_option(
'\0', "broadcast-speed-catchain",
"multiplier for broadcast speed in catchain overlays (experimental, default is 1.0, which is ~300 KB/s)",
[&](td::Slice s) -> td::Status {
auto v = td::to_double(s);
if (v <= 0.0) {
return td::Status::Error("broadcast-speed-catchain should be positive");
}
acts.push_back(
[&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_catchain, v); });
return td::Status::OK();
});
p.add_checked_option(
'\0', "broadcast-speed-public",
"multiplier for broadcast speed in public shard overlays (experimental, default is 1.0, which is ~300 KB/s)",
[&](td::Slice s) -> td::Status {
auto v = td::to_double(s);
if (v <= 0.0) {
return td::Status::Error("broadcast-speed-public should be positive");
}
acts.push_back(
[&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_public, v); });
return td::Status::OK();
});
p.add_checked_option(
'\0', "broadcast-speed-private",
"multiplier for broadcast speed in private block overlays (experimental, default is 1.0, which is ~300 KB/s)",
[&](td::Slice s) -> td::Status {
auto v = td::to_double(s);
if (v <= 0.0) {
return td::Status::Error("broadcast-speed-private should be positive");
}
acts.push_back(
[&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_private, v); });
return td::Status::OK();
});
auto S = p.run(argc, argv);
if (S.is_error()) {
LOG(ERROR) << "failed to parse options: " << S.move_as_error();

16
validator-engine/validator-engine.hpp
View file

@ -248,6 +248,10 @@ class ValidatorEngine : public td::actor::Actor {
std::string validator_telemetry_filename_;
bool not_all_shards_ = false;
std::vector<ton::ShardIdFull> add_shard_cmds_;
bool state_serializer_disabled_flag_ = false;
double broadcast_speed_multiplier_catchain_ = 1.0;
double broadcast_speed_multiplier_public_ = 1.0;
double broadcast_speed_multiplier_private_ = 1.0;
std::set<ton::CatchainSeqno> unsafe_catchains_;
std::map<ton::BlockSeqno, std::pair<ton::CatchainSeqno, td::uint32>> unsafe_catchain_rotations_;
@ -345,6 +349,18 @@ class ValidatorEngine : public td::actor::Actor {
void add_shard_cmd(ton::ShardIdFull shard) {
add_shard_cmds_.push_back(shard);
}
void set_state_serializer_disabled_flag() {
state_serializer_disabled_flag_ = true;
}
void set_broadcast_speed_multiplier_catchain(double value) {
broadcast_speed_multiplier_catchain_ = value;
}
void set_broadcast_speed_multiplier_public(double value) {
broadcast_speed_multiplier_public_ = value;
}
void set_broadcast_speed_multiplier_private(double value) {
broadcast_speed_multiplier_private_ = value;
}
void start_up() override;
ValidatorEngine() {

24
validator/db/archive-manager.cpp
View file

@ -1196,6 +1196,30 @@ void ArchiveManager::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
}
}
void ArchiveManager::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
std::vector<std::pair<std::string, std::string>> stats;
{
std::map<BlockSeqno, td::uint64> states;
for (auto &[key, file] : perm_states_) {
BlockSeqno seqno = key.first;
auto r_stat = td::stat(db_root_ + "/archive/states/" + file.filename_short());
if (r_stat.is_error()) {
LOG(WARNING) << "Cannot stat persistent state file " << file.filename_short() << " : " << r_stat.move_as_error();
} else {
states[seqno] += r_stat.move_as_ok().size_;
}
}
td::StringBuilder sb;
for (auto &[seqno, size] : states) {
sb << seqno << ":" << td::format::as_size(size) << " ";
}
if (!sb.as_cslice().empty()) {
stats.emplace_back("persistent_states", sb.as_cslice().str());
}
}
promise.set_value(std::move(stats));
}
void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise) {
index_->begin_transaction().ensure();
td::MultiPromise mp;

2
validator/db/archive-manager.hpp
View file

@ -81,6 +81,8 @@ class ArchiveManager : public td::actor::Actor {
cur_shard_split_depth_ = value;
}
void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise);
static constexpr td::uint32 archive_size() {
return 20000;
}

19
validator/db/celldb.cpp
View file

@ -158,6 +158,17 @@ void CellDbIn::start_up() {
},
td::Timestamp::now());
}
{
std::string key = "stats.last_deleted_mc_seqno", value;
auto R = cell_db_->get(td::as_slice(key), value);
R.ensure();
if (R.ok() == td::KeyValue::GetStatus::Ok) {
auto r_value = td::to_integer_safe<BlockSeqno>(value);
r_value.ensure();
last_deleted_mc_state_ = r_value.move_as_ok();
}
}
}
void CellDbIn::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
@ -452,6 +463,11 @@ void CellDbIn::gc_cont2(BlockHandle handle) {
cell_db_->erase(get_key(key_hash)).ensure();
set_block(F.prev, std::move(P));
set_block(F.next, std::move(N));
if (handle->id().is_masterchain()) {
last_deleted_mc_state_ = handle->id().seqno();
std::string key = "stats.last_deleted_mc_seqno", value = td::to_string(last_deleted_mc_state_);
cell_db_->set(td::as_slice(key), td::as_slice(value));
}
cell_db_->commit_write_batch().ensure();
alarm_timestamp() = td::Timestamp::now();
timer_write_batch.reset();
@ -475,9 +491,6 @@ void CellDbIn::gc_cont2(BlockHandle handle) {
if (!opts_->get_disable_rocksdb_stats()) {
cell_db_statistics_.gc_cell_time_.insert(timer.elapsed() * 1e6);
}
if (handle->id().is_masterchain()) {
last_deleted_mc_state_ = handle->id().seqno();
}
LOG(DEBUG) << "Deleted state " << handle->id().to_str();
timer_finish.reset();
timer_all.reset();

1
validator/db/rootdb.cpp
View file

@ -438,6 +438,7 @@ void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise<bool> promise) {
void RootDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
auto merger = StatsMerger::create(std::move(promise));
td::actor::send_closure(cell_db_, &CellDb::prepare_stats, merger.make_promise("celldb."));
td::actor::send_closure(archive_db_, &ArchiveManager::prepare_stats, merger.make_promise("archive."));
}
void RootDb::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise) {

7
validator/downloaders/download-state.cpp
View file

@ -38,6 +38,7 @@ DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt mastercha
}
void DownloadShardState::start_up() {
status_ = ProcessStatus(manager_, "process.download_state");
alarm_timestamp() = timeout_;
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
@ -81,6 +82,7 @@ void DownloadShardState::download_state() {
});
td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, block_id_, priority_,
std::move(P));
status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof");
}
void DownloadShardState::downloaded_proof_link(td::BufferSlice data) {
@ -123,6 +125,7 @@ void DownloadShardState::checked_proof_link() {
td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_,
masterchain_block_id_, priority_, std::move(P));
}
status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state");
}
void DownloadShardState::download_zero_state() {
@ -152,6 +155,7 @@ void DownloadShardState::downloaded_zero_state(td::BufferSlice data) {
}
void DownloadShardState::downloaded_shard_state(td::BufferSlice data) {
status_.set_status(PSTRING() << block_id_.id.to_str() << " : processing downloaded state");
auto S = create_shard_state(block_id_, data.clone());
if (S.is_error()) {
fail_handler(actor_id(this), S.move_as_error());
@ -174,6 +178,7 @@ void DownloadShardState::downloaded_shard_state(td::BufferSlice data) {
}
void DownloadShardState::checked_shard_state() {
status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state file");
LOG(WARNING) << "checked shard state " << block_id_.to_str();
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
R.ensure();
@ -189,6 +194,7 @@ void DownloadShardState::checked_shard_state() {
}
void DownloadShardState::written_shard_state_file() {
status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state to celldb");
LOG(WARNING) << "written shard state file " << block_id_.to_str();
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
R.ensure();
@ -198,6 +204,7 @@ void DownloadShardState::written_shard_state_file() {
}
void DownloadShardState::written_shard_state(td::Ref<ShardState> state) {
status_.set_status(PSTRING() << block_id_.id.to_str() << " : finishing");
state_ = std::move(state);
handle_->set_unix_time(state_->get_unix_time());
handle_->set_is_key_block(block_id_.is_masterchain());

3
validator/downloaders/download-state.hpp
View file

@ -19,6 +19,7 @@
#pragma once
#include "validator/interfaces/validator-manager.h"
#include "stats-provider.h"
namespace ton {
@ -67,6 +68,8 @@ class DownloadShardState : public td::actor::Actor {
td::BufferSlice data_;
td::Ref<ShardState> state_;
ProcessStatus status_;
};
} // namespace validator

16
validator/full-node-private-overlay.cpp
View file

@ -266,8 +266,11 @@ void FullNodePrivateBlockOverlay::init() {
overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(),
overlay::CertificateFlags::AllowFec | overlay::CertificateFlags::Trusted,
{}};
td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(),
nodes_, std::make_unique<Callback>(actor_id(this)), rules, R"({ "type": "private-blocks" })");
overlay::OverlayOptions overlay_options;
overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_;
td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(),
nodes_, std::make_unique<Callback>(actor_id(this)), rules, R"({ "type": "private-blocks" })",
overlay_options);
td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_);
td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_);
@ -368,7 +371,7 @@ void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice
}
void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) {
if (!inited_ || config_.ext_messages_broadcast_disabled_) {
if (!inited_ || opts_.config_.ext_messages_broadcast_disabled_) {
return;
}
VLOG(FULL_NODE_DEBUG) << "Sending external message to custom overlay \"" << name_ << "\"";
@ -474,10 +477,13 @@ void FullNodeCustomOverlay::init() {
authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size();
}
overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)};
overlay::OverlayOptions overlay_options;
overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_;
td::actor::send_closure(
overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), nodes_,
overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), nodes_,
std::make_unique<Callback>(actor_id(this)), rules,
PSTRING() << R"({ "type": "custom-overlay", "name": ")" << td::format::Escaped{name_} << R"(" })");
PSTRING() << R"({ "type": "custom-overlay", "name": ")" << td::format::Escaped{name_} << R"(" })",
overlay_options);
td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_);
td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_);

16
validator/full-node-private-overlay.hpp
View file

@ -50,14 +50,14 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor {
void collect_validator_telemetry(std::string filename);
void set_config(FullNodeConfig config) {
config_ = std::move(config);
opts_.config_ = std::move(config);
}
void start_up() override;
void tear_down() override;
FullNodePrivateBlockOverlay(adnl::AdnlNodeIdShort local_id, std::vector<adnl::AdnlNodeIdShort> nodes,
FileHash zero_state_file_hash, FullNodeConfig config,
FileHash zero_state_file_hash, FullNodeOptions opts,
td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2,
td::actor::ActorId<overlay::Overlays> overlays,
@ -66,7 +66,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor {
: local_id_(local_id)
, nodes_(std::move(nodes))
, zero_state_file_hash_(zero_state_file_hash)
, config_(config)
, opts_(opts)
, keyring_(keyring)
, adnl_(adnl)
, rldp_(rldp)
@ -80,7 +80,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor {
adnl::AdnlNodeIdShort local_id_;
std::vector<adnl::AdnlNodeIdShort> nodes_;
FileHash zero_state_file_hash_;
FullNodeConfig config_;
FullNodeOptions opts_;
bool enable_compression_ = true;
td::actor::ActorId<keyring::Keyring> keyring_;
@ -126,14 +126,14 @@ class FullNodeCustomOverlay : public td::actor::Actor {
td::BufferSlice data);
void set_config(FullNodeConfig config) {
config_ = std::move(config);
opts_.config_ = std::move(config);
}
void start_up() override;
void tear_down() override;
FullNodeCustomOverlay(adnl::AdnlNodeIdShort local_id, CustomOverlayParams params, FileHash zero_state_file_hash,
FullNodeConfig config, td::actor::ActorId<keyring::Keyring> keyring,
FullNodeOptions opts, td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<rldp::Rldp> rldp,
td::actor::ActorId<rldp2::Rldp> rldp2, td::actor::ActorId<overlay::Overlays> overlays,
td::actor::ActorId<ValidatorManagerInterface> validator_manager,
@ -144,7 +144,7 @@ class FullNodeCustomOverlay : public td::actor::Actor {
, msg_senders_(std::move(params.msg_senders_))
, block_senders_(std::move(params.block_senders_))
, zero_state_file_hash_(zero_state_file_hash)
, config_(config)
, opts_(opts)
, keyring_(keyring)
, adnl_(adnl)
, rldp_(rldp)
@ -161,7 +161,7 @@ class FullNodeCustomOverlay : public td::actor::Actor {
std::map<adnl::AdnlNodeIdShort, int> msg_senders_;
std::set<adnl::AdnlNodeIdShort> block_senders_;
FileHash zero_state_file_hash_;
FullNodeConfig config_;
FullNodeOptions opts_;
td::actor::ActorId<keyring::Keyring> keyring_;
td::actor::ActorId<adnl::Adnl> adnl_;

13
validator/full-node-shard.cpp
View file

@ -105,6 +105,7 @@ void FullNodeShardImpl::create_overlay() {
};
overlay::OverlayOptions opts;
opts.announce_self_ = active_;
opts.broadcast_speed_multiplier_ = opts_.public_broadcast_speed_multiplier_;
td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_ex, adnl_id_, overlay_id_full_.clone(),
std::make_unique<Callback>(actor_id(this)), rules_,
PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard()
@ -132,7 +133,7 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad
if (!processed_ext_msg_broadcasts_.insert(hash).second) {
return promise.set_error(td::Status::Error("duplicate external message broadcast"));
}
if (config_.ext_messages_broadcast_disabled_) {
if (opts_.config_.ext_messages_broadcast_disabled_) {
promise.set_error(td::Status::Error("rebroadcasting external messages is disabled"));
promise = [manager = validator_manager_, message = q->message_->data_.clone()](td::Result<td::Unit> R) mutable {
if (R.is_ok()) {
@ -850,7 +851,7 @@ void FullNodeShardImpl::send_ihr_message(td::BufferSlice data) {
}
void FullNodeShardImpl::send_external_message(td::BufferSlice data) {
if (config_.ext_messages_broadcast_disabled_) {
if (opts_.config_.ext_messages_broadcast_disabled_) {
return;
}
if (!client_.empty()) {
@ -1379,7 +1380,7 @@ void FullNodeShardImpl::get_stats_extra(td::Promise<std::string> promise) {
}
FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id,
FileHash zero_state_file_hash, FullNodeConfig config,
FileHash zero_state_file_hash, FullNodeOptions opts,
td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2,
td::actor::ActorId<overlay::Overlays> overlays,
@ -1399,17 +1400,17 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id,
, client_(client)
, full_node_(full_node)
, active_(active)
, config_(config) {
, opts_(opts) {
}
td::actor::ActorOwn<FullNodeShard> FullNodeShard::create(
ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash,
FullNodeConfig config, td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
FullNodeOptions opts, td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2,
td::actor::ActorId<overlay::Overlays> overlays, td::actor::ActorId<ValidatorManagerInterface> validator_manager,
td::actor::ActorId<adnl::AdnlExtClient> client, td::actor::ActorId<FullNode> full_node, bool active) {
return td::actor::create_actor<FullNodeShardImpl>(PSTRING() << "tonnode" << shard.to_str(), shard, local_id, adnl_id,
zero_state_file_hash, config, keyring, adnl, rldp, rldp2, overlays,
zero_state_file_hash, opts, keyring, adnl, rldp, rldp2, overlays,
validator_manager, client, full_node, active);
}

2
validator/full-node-shard.h
View file

@ -76,7 +76,7 @@ class FullNodeShard : public td::actor::Actor {
static td::actor::ActorOwn<FullNodeShard> create(
ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash,
FullNodeConfig config, td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
FullNodeOptions opts, td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2,
td::actor::ActorId<overlay::Overlays> overlays, td::actor::ActorId<ValidatorManagerInterface> validator_manager,
td::actor::ActorId<adnl::AdnlExtClient> client, td::actor::ActorId<FullNode> full_node, bool active);

6
validator/full-node-shard.hpp
View file

@ -86,7 +86,7 @@ class FullNodeShardImpl : public FullNodeShard {
void set_active(bool active) override;
void set_config(FullNodeConfig config) override {
config_ = config;
opts_.config_ = config;
}
void try_get_next_block(td::Timestamp timestamp, td::Promise<ReceivedBlock> promise);
@ -225,7 +225,7 @@ class FullNodeShardImpl : public FullNodeShard {
}
FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id,
FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId<keyring::Keyring> keyring,
FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<rldp::Rldp> rldp,
td::actor::ActorId<rldp2::Rldp> rldp2, td::actor::ActorId<overlay::Overlays> overlays,
td::actor::ActorId<ValidatorManagerInterface> validator_manager,
@ -272,7 +272,7 @@ class FullNodeShardImpl : public FullNodeShard {
bool active_;
FullNodeConfig config_;
FullNodeOptions opts_;
std::set<td::Bits256> my_ext_msg_broadcasts_;
std::set<td::Bits256> processed_ext_msg_broadcasts_;

16
validator/full-node.cpp
View file

@ -149,7 +149,7 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise<td:
}
void FullNodeImpl::set_config(FullNodeConfig config) {
config_ = config;
opts_.config_ = config;
for (auto& s : shards_) {
if (!s.second.actor.empty()) {
td::actor::send_closure(s.second.actor, &FullNodeShard::set_config, config);
@ -288,7 +288,7 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref<MasterchainState> state, std
void FullNodeImpl::update_shard_actor(ShardIdFull shard, bool active) {
ShardInfo &info = shards_[shard];
if (info.actor.empty()) {
info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, adnl_, rldp_,
info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, opts_, keyring_, adnl_, rldp_,
rldp2_, overlays_, validator_manager_, client_, actor_id(this), active);
if (!all_validators_.empty()) {
td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_);
@ -816,7 +816,7 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) {
nodes.push_back(p.second);
}
private_block_overlays_[key] = td::actor::create_actor<FullNodePrivateBlockOverlay>(
"BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_,
"BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, opts_, keyring_,
adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this));
update_validator_telemetry_collector();
}
@ -834,7 +834,7 @@ void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) {
old_actors.erase(it);
} else {
overlay.actors_[local_id] = td::actor::create_actor<FullNodeCustomOverlay>(
"CustomOverlay", local_id, params, zero_state_file_hash_, config_, keyring_, adnl_, rldp_, rldp2_,
"CustomOverlay", local_id, params, zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, rldp2_,
overlays_, validator_manager_, actor_id(this));
}
}
@ -893,7 +893,7 @@ void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const Block
}
FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash,
FullNodeConfig config, td::actor::ActorId<keyring::Keyring> keyring,
FullNodeOptions opts, td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<rldp::Rldp> rldp,
td::actor::ActorId<rldp2::Rldp> rldp2, td::actor::ActorId<dht::Dht> dht,
td::actor::ActorId<overlay::Overlays> overlays,
@ -913,16 +913,16 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id
, client_(client)
, db_root_(db_root)
, started_promise_(std::move(started_promise))
, config_(config) {
, opts_(opts) {
}
td::actor::ActorOwn<FullNode> FullNode::create(
ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config,
ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts,
td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2, td::actor::ActorId<dht::Dht> dht,
td::actor::ActorId<overlay::Overlays> overlays, td::actor::ActorId<ValidatorManagerInterface> validator_manager,
td::actor::ActorId<adnl::AdnlExtClient> client, std::string db_root, td::Promise<td::Unit> started_promise) {
return td::actor::create_actor<FullNodeImpl>("fullnode", local_id, adnl_id, zero_state_file_hash, config, keyring,
return td::actor::create_actor<FullNodeImpl>("fullnode", local_id, adnl_id, zero_state_file_hash, opts, keyring,
adnl, rldp, rldp2, dht, overlays, validator_manager, client, db_root,
std::move(started_promise));
}

8
validator/full-node.h
View file

@ -55,6 +55,12 @@ struct FullNodeConfig {
bool ext_messages_broadcast_disabled_ = false;
};
struct FullNodeOptions {
FullNodeConfig config_;
double public_broadcast_speed_multiplier_ = 1.0;
double private_broadcast_speed_multiplier_ = 1.0;
};
struct CustomOverlayParams {
std::string name_;
std::vector<adnl::AdnlNodeIdShort> nodes_;
@ -114,7 +120,7 @@ class FullNode : public td::actor::Actor {
static constexpr td::int32 MAX_FAST_SYNC_OVERLAY_CLIENTS = 5000; // TODO: set lower limit (high limit for testing)
static td::actor::ActorOwn<FullNode> create(
ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config,
ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts,
td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2, td::actor::ActorId<dht::Dht> dht,
td::actor::ActorId<overlay::Overlays> overlays, td::actor::ActorId<ValidatorManagerInterface> validator_manager,

4
validator/full-node.hpp
View file

@ -110,7 +110,7 @@ class FullNodeImpl : public FullNode {
void start_up() override;
FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash,
FullNodeConfig config, td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
FullNodeOptions opts, td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2,
td::actor::ActorId<dht::Dht> dht, td::actor::ActorId<overlay::Overlays> overlays,
td::actor::ActorId<ValidatorManagerInterface> validator_manager,
@ -154,7 +154,7 @@ class FullNodeImpl : public FullNode {
std::map<adnl::AdnlNodeIdShort, int> local_collator_nodes_;
td::Promise<td::Unit> started_promise_;
FullNodeConfig config_;
FullNodeOptions opts_;
// Private overlays:
// Old overlays - one private overlay for all validators

20
validator/impl/accept-block.cpp
View file

@ -310,8 +310,11 @@ bool AcceptBlockQuery::create_new_proof() {
}
// 10. check resulting object
if (!block::gen::t_BlockProof.validate_ref(bs_cell)) {
block::gen::t_BlockProof.print_ref(std::cerr, bs_cell);
vm::load_cell_slice(bs_cell).print_rec(std::cerr);
FLOG(WARNING) {
sb << "BlockProof object just created failed to pass automated consistency checks: ";
block::gen::t_BlockProof.print_ref(sb, bs_cell);
vm::load_cell_slice(bs_cell).print_rec(sb);
};
return fatal_error("BlockProof object just created failed to pass automated consistency checks");
}
// 11. create a proof object from this cell
@ -860,15 +863,12 @@ bool AcceptBlockQuery::create_top_shard_block_description() {
&& (root.is_null() || cb.store_ref_bool(std::move(root))) && cb.finalize_to(td_cell))) {
return fatal_error("cannot serialize ShardTopBlockDescription for the newly-accepted block "s + id_.to_str());
}
if (false) {
// debug output
std::cerr << "new ShardTopBlockDescription: ";
block::gen::t_TopBlockDescr.print_ref(std::cerr, td_cell);
vm::load_cell_slice(td_cell).print_rec(std::cerr);
}
if (!block::gen::t_TopBlockDescr.validate_ref(td_cell)) {
block::gen::t_TopBlockDescr.print_ref(std::cerr, td_cell);
vm::load_cell_slice(td_cell).print_rec(std::cerr);
FLOG(WARNING) {
sb << "just created ShardTopBlockDescription is invalid: ";
block::gen::t_TopBlockDescr.print_ref(sb, td_cell);
vm::load_cell_slice(td_cell).print_rec(sb);
};
return fatal_error("just created ShardTopBlockDescription for "s + id_.to_str() + " is invalid");
}
auto res = vm::std_boc_serialize(td_cell, 0);

4
validator/impl/collator-impl.h
View file

@ -53,7 +53,7 @@ class Collator final : public td::actor::Actor {
using LtCellRef = block::LtCellRef;
using NewOutMsg = block::NewOutMsg;
const ShardIdFull shard_;
ton::BlockId new_id;
ton::BlockId new_id{workchainInvalid, 0, 0};
bool busy_{false};
bool before_split_{false};
bool after_split_{false};
@ -392,6 +392,8 @@ class Collator final : public td::actor::Actor {
td::Timer work_timer_{true};
td::ThreadCpuTimer cpu_work_timer_{true};
CollationStats stats_;
void finalize_stats();
};
} // namespace validator

334
validator/impl/collator.cpp
View file

@ -54,16 +54,6 @@ static constexpr int HIGH_PRIORITY_EXTERNAL = 10; // don't skip high priority e
static constexpr int MAX_ATTEMPTS = 5;
#define DBG(__n) dbg(__n)&&
#define DSTART int __dcnt = 0;
#define DEB DBG(++__dcnt)
static inline bool dbg(int c) TD_UNUSED;
static inline bool dbg(int c) {
std::cerr << '[' << (char)('0' + c / 10) << (char)('0' + c % 10) << ']';
return true;
}
/**
* Constructs a Collator object.
*
@ -381,6 +371,9 @@ bool Collator::fatal_error(td::Status error) {
} else {
LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error;
LOG(INFO) << perf_log_;
finalize_stats();
stats_.status = error.clone();
td::actor::send_closure(manager, &ValidatorManager::log_collate_query_stats, std::move(stats_));
main_promise(std::move(error));
}
busy_ = false;
@ -812,8 +805,6 @@ bool Collator::unpack_last_mc_state() {
<< " (upgrade validator software?)";
}
// TODO: extract start_lt and end_lt from prev_mc_block as well
// std::cerr << " block::gen::ShardState::print_ref(mc_state_root) = ";
// block::gen::t_ShardState.print_ref(std::cerr, mc_state_root, 2);
return true;
}
@ -1000,8 +991,10 @@ void Collator::got_neighbor_msg_queue(unsigned i, Ref<OutMsgQueueProof> res) {
// unpack ProcessedUpto
LOG(DEBUG) << "unpacking ProcessedUpto of neighbor " << descr.blk_.to_str();
if (verbosity >= 2) {
block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info);
qinfo.proc_info->print_rec(std::cerr);
FLOG(INFO) {
block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info);
qinfo.proc_info->print_rec(sb);
};
}
descr.processed_upto = block::MsgProcessedUptoCollection::unpack(descr.shard(), qinfo.proc_info);
if (!descr.processed_upto) {
@ -1879,13 +1872,12 @@ bool Collator::import_new_shard_top_blocks() {
if (tb_act) {
shard_conf_adjusted_ = true;
}
if (tb_act) {
LOG(INFO) << "updated shard block configuration: " << tb_act << " new top shard blocks";
if (verbosity >= 3) {
LOG(INFO) << "updated shard block configuration to ";
if (tb_act && verbosity >= 0) { // DEBUG
FLOG(INFO) {
sb << "updated shard block configuration to ";
auto csr = shard_conf_->get_root_csr();
block::gen::t_ShardHashes.print(std::cerr, csr.write());
}
block::gen::t_ShardHashes.print(sb, csr);
};
}
block::gen::ShardFeeCreated::Record fc;
if (!(tlb::csr_unpack(fees_import_dict_->get_root_extra(),
@ -2422,10 +2414,12 @@ bool Collator::dequeue_message(Ref<vm::Cell> msg_envelope, ton::LogicalTime deli
bool Collator::out_msg_queue_cleanup() {
LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors";
if (verbosity >= 2) {
auto rt = out_msg_queue_->get_root();
std::cerr << "old out_msg_queue is ";
block::gen::t_OutMsgQueue.print(std::cerr, *rt);
rt->print_rec(std::cerr);
FLOG(INFO) {
auto rt = out_msg_queue_->get_root();
sb << "old out_msg_queue is ";
block::gen::t_OutMsgQueue.print(sb, rt);
rt->print_rec(sb);
};
}
if (after_merge_) {
@ -2587,10 +2581,12 @@ bool Collator::out_msg_queue_cleanup() {
<< out_msg_queue_size_;
}
if (verbosity >= 2) {
auto rt = out_msg_queue_->get_root();
std::cerr << "new out_msg_queue is ";
block::gen::t_OutMsgQueue.print(std::cerr, *rt);
rt->print_rec(std::cerr);
FLOG(INFO) {
auto rt = out_msg_queue_->get_root();
sb << "new out_msg_queue is ";
block::gen::t_OutMsgQueue.print(sb, rt);
rt->print_rec(sb);
};
}
return register_out_msg_queue_op(true);
}
@ -2689,19 +2685,27 @@ bool Collator::combine_account_transactions() {
auto cell = cb.finalize();
auto csr = vm::load_cell_slice_ref(cell);
if (verbosity > 2) {
std::cerr << "new AccountBlock for " << z.first.to_hex() << ": ";
block::gen::t_AccountBlock.print_ref(std::cerr, cell);
csr->print_rec(std::cerr);
FLOG(INFO) {
sb << "new AccountBlock for " << z.first.to_hex() << ": ";
block::gen::t_AccountBlock.print_ref(sb, cell);
csr->print_rec(sb);
};
}
if (!block::gen::t_AccountBlock.validate_ref(100000, cell)) {
block::gen::t_AccountBlock.print_ref(std::cerr, cell);
csr->print_rec(std::cerr);
FLOG(WARNING) {
sb << "AccountBlock failed to pass automatic validation tests: ";
block::gen::t_AccountBlock.print_ref(sb, cell);
csr->print_rec(sb);
};
return fatal_error(std::string{"new AccountBlock for "} + z.first.to_hex() +
" failed to pass automatic validation tests");
}
if (!block::tlb::t_AccountBlock.validate_ref(100000, cell)) {
block::gen::t_AccountBlock.print_ref(std::cerr, cell);
csr->print_rec(std::cerr);
FLOG(WARNING) {
sb << "AccountBlock failed to pass handwritten validation tests: ";
block::gen::t_AccountBlock.print_ref(sb, cell);
csr->print_rec(sb);
};
return fatal_error(std::string{"new AccountBlock for "} + z.first.to_hex() +
" failed to pass handwritten validation tests");
}
@ -2726,8 +2730,10 @@ bool Collator::combine_account_transactions() {
} else if (acc.status == block::Account::acc_nonexist) {
// account deleted
if (verbosity > 2) {
std::cerr << "deleting account " << acc.addr.to_hex() << " with empty new value ";
block::gen::t_Account.print_ref(std::cerr, acc.total_state);
FLOG(INFO) {
sb << "deleting account " << acc.addr.to_hex() << " with empty new value ";
block::gen::t_Account.print_ref(sb, acc.total_state);
};
}
if (account_dict->lookup_delete(acc.addr).is_null()) {
return fatal_error(std::string{"cannot delete account "} + acc.addr.to_hex() + " from ShardAccounts");
@ -2735,8 +2741,10 @@ bool Collator::combine_account_transactions() {
} else {
// existing account modified
if (verbosity > 4) {
std::cerr << "modifying account " << acc.addr.to_hex() << " to ";
block::gen::t_Account.print_ref(std::cerr, acc.total_state);
FLOG(INFO) {
sb << "modifying account " << acc.addr.to_hex() << " to ";
block::gen::t_Account.print_ref(sb, acc.total_state);
};
}
if (!(cb.store_ref_bool(acc.total_state) // account_descr$_ account:^Account
&& cb.store_bits_bool(acc.last_trans_hash_) // last_trans_hash:bits256
@ -2759,9 +2767,11 @@ bool Collator::combine_account_transactions() {
return fatal_error("cannot serialize ShardAccountBlocks");
}
if (verbosity > 2) {
std::cerr << "new ShardAccountBlocks: ";
block::gen::t_ShardAccountBlocks.print_ref(std::cerr, shard_account_blocks_);
vm::load_cell_slice(shard_account_blocks_).print_rec(std::cerr);
FLOG(INFO) {
sb << "new ShardAccountBlocks: ";
block::gen::t_ShardAccountBlocks.print_ref(sb, shard_account_blocks_);
vm::load_cell_slice(shard_account_blocks_).print_rec(sb);
};
}
if (!block::gen::t_ShardAccountBlocks.validate_ref(100000, shard_account_blocks_)) {
return fatal_error("new ShardAccountBlocks failed to pass automatic validity tests");
@ -2771,9 +2781,11 @@ bool Collator::combine_account_transactions() {
}
auto shard_accounts = account_dict->get_root();
if (verbosity > 2) {
std::cerr << "new ShardAccounts: ";
block::gen::t_ShardAccounts.print(std::cerr, *shard_accounts);
shard_accounts->print_rec(std::cerr);
FLOG(INFO) {
sb << "new ShardAccounts: ";
block::gen::t_ShardAccounts.print(sb, shard_accounts);
shard_accounts->print_rec(sb);
};
}
if (verify >= 2) {
LOG(INFO) << "verifying new ShardAccounts";
@ -2824,7 +2836,9 @@ bool Collator::create_special_transaction(block::CurrencyCollection amount, Ref<
addr.to_hex());
}
if (verbosity >= 4) {
block::gen::t_Message_Any.print_ref(std::cerr, msg);
FLOG(INFO) {
block::gen::t_Message_Any.print_ref(sb, msg);
};
}
CHECK(block::gen::t_Message_Any.validate_ref(msg));
CHECK(block::tlb::t_Message.validate_ref(msg));
@ -3328,8 +3342,10 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R
Ref<vm::Cell> msg_env;
CHECK(block::tlb::pack_cell(msg_env, msg_env_rec));
if (verbosity > 2) {
std::cerr << "new (processed outbound) message envelope: ";
block::gen::t_MsgEnvelope.print_ref(std::cerr, msg_env);
FLOG(INFO) {
sb << "new (processed outbound) message envelope: ";
block::gen::t_MsgEnvelope.print_ref(sb, msg_env);
};
}
// 3. create InMsg, referring to this MsgEnvelope and this Transaction
vm::CellBuilder cb;
@ -3451,16 +3467,20 @@ bool Collator::enqueue_transit_message(Ref<vm::Cell> msg, Ref<vm::Cell> old_msg_
Ref<vm::Cell> out_msg = cb.finalize();
// 4.1. insert OutMsg into OutMsgDescr
if (verbosity > 2) {
std::cerr << "OutMsg for a transit message: ";
block::gen::t_OutMsg.print_ref(std::cerr, out_msg);
FLOG(INFO) {
sb << "OutMsg for a transit message: ";
block::gen::t_OutMsg.print_ref(sb, out_msg);
};
}
if (!insert_out_msg(out_msg)) {
return fatal_error("cannot insert a new OutMsg into OutMsgDescr");
}
// 4.2. insert InMsg into InMsgDescr
if (verbosity > 2) {
std::cerr << "InMsg for a transit message: ";
block::gen::t_InMsg.print_ref(std::cerr, in_msg);
FLOG(INFO) {
sb << "InMsg for a transit message: ";
block::gen::t_InMsg.print_ref(sb, in_msg);
};
}
if (!insert_in_msg(in_msg)) {
return fatal_error("cannot insert a new InMsg into InMsgDescr");
@ -3519,7 +3539,10 @@ bool Collator::precheck_inbound_message(Ref<vm::CellSlice> enq_msg, ton::Logical
if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 ||
(enqueued_lt = enq_msg->prefetch_ulong(64)) < /* 0 */ 1 * lt) { // DEBUG
if (enq_msg.not_null()) {
block::gen::t_EnqueuedMsg.print(std::cerr, *enq_msg);
FLOG(WARNING) {
sb << "inbound internal message is not a valid EnqueuedMsg: ";
block::gen::t_EnqueuedMsg.print(sb, enq_msg);
};
}
LOG(ERROR) << "inbound internal message is not a valid EnqueuedMsg (created lt " << lt << ", enqueued "
<< enqueued_lt << ")";
@ -3780,14 +3803,18 @@ bool Collator::process_inbound_internal_messages() {
LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex()
<< ") from neighbor #" << kv->source;
if (verbosity > 2) {
std::cerr << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg=";
block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg));
FLOG(INFO) {
sb << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg=";
block::gen::t_EnqueuedMsg.print(sb, kv->msg);
};
}
if (!process_inbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source))) {
if (verbosity > 1) {
std::cerr << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex()
<< " msg=";
block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg));
FLOG(INFO) {
sb << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex()
<< " msg=";
block::gen::t_EnqueuedMsg.print(sb, kv->msg);
};
}
return fatal_error("error processing inbound internal message");
}
@ -4074,7 +4101,10 @@ bool Collator::process_deferred_message(Ref<vm::CellSlice> enq_msg, StdSmcAddres
LogicalTime enqueued_lt = 0;
if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 || (enqueued_lt = enq_msg->prefetch_ulong(64)) != lt) {
if (enq_msg.not_null()) {
block::gen::t_EnqueuedMsg.print(std::cerr, *enq_msg);
FLOG(WARNING) {
sb << "internal message in DispatchQueue is not a valid EnqueuedMsg: ";
block::gen::t_EnqueuedMsg.print(sb, enq_msg);
};
}
LOG(ERROR) << "internal message in DispatchQueue is not a valid EnqueuedMsg (created lt " << lt << ", enqueued "
<< enqueued_lt << ")";
@ -4176,8 +4206,10 @@ bool Collator::process_deferred_message(Ref<vm::CellSlice> enq_msg, StdSmcAddres
*/
bool Collator::insert_in_msg(Ref<vm::Cell> in_msg) {
if (verbosity > 2) {
std::cerr << "InMsg being inserted into InMsgDescr: ";
block::gen::t_InMsg.print_ref(std::cerr, in_msg);
FLOG(INFO) {
sb << "InMsg being inserted into InMsgDescr: ";
block::gen::t_InMsg.print_ref(sb, in_msg);
};
}
auto cs = load_cell_slice(in_msg);
if (!cs.size_refs()) {
@ -4218,8 +4250,10 @@ bool Collator::insert_in_msg(Ref<vm::Cell> in_msg) {
*/
bool Collator::insert_out_msg(Ref<vm::Cell> out_msg) {
if (verbosity > 2) {
std::cerr << "OutMsg being inserted into OutMsgDescr: ";
block::gen::t_OutMsg.print_ref(std::cerr, out_msg);
FLOG(INFO) {
sb << "OutMsg being inserted into OutMsgDescr: ";
block::gen::t_OutMsg.print_ref(sb, out_msg);
};
}
auto cs = load_cell_slice(out_msg);
if (!cs.size_refs()) {
@ -4315,8 +4349,10 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema
}
// 4. insert OutMsg into OutMsgDescr
if (verbosity > 2) {
std::cerr << "OutMsg for a newly-generated message: ";
block::gen::t_OutMsg.print_ref(std::cerr, out_msg);
FLOG(INFO) {
sb << "OutMsg for a newly-generated message: ";
block::gen::t_OutMsg.print_ref(sb, out_msg);
};
}
if (!insert_out_msg(out_msg)) {
return fatal_error("cannot insert a new OutMsg into OutMsgDescr");
@ -4609,9 +4645,12 @@ bool Collator::create_mc_state_extra() {
bool ignore_cfg_changes = false;
Ref<vm::Cell> cfg0;
if (!block::valid_config_data(cfg_smc_config, config_addr, true, true, old_mparams_)) {
block::gen::t_Hashmap_32_Ref_Cell.print_ref(std::cerr, cfg_smc_config);
LOG(ERROR) << "configuration smart contract "s + config_addr.to_hex() +
" contains an invalid configuration in its data, IGNORING CHANGES";
FLOG(WARNING) {
sb << "ignored configuration: ";
block::gen::t_Hashmap_32_Ref_Cell.print_ref(sb, cfg_smc_config);
};
ignore_cfg_changes = true;
} else {
cfg0 = cfg_dict.lookup_ref(td::BitArray<32>{(long long)0});
@ -4649,34 +4688,26 @@ bool Collator::create_mc_state_extra() {
return fatal_error(wset_res.move_as_error());
}
bool update_shard_cc = is_key_block_ || (now_ / ccvc.shard_cc_lifetime > prev_now_ / ccvc.shard_cc_lifetime);
// temp debug
if (verbosity >= 3 * 1) {
auto csr = shard_conf_->get_root_csr();
LOG(INFO) << "new shard configuration before post-processing is";
std::ostringstream os;
csr->print_rec(os);
block::gen::t_ShardHashes.print(os, csr.write());
LOG(INFO) << os.str();
}
// end (temp debug)
if (!update_shard_config(wset_res.move_as_ok(), ccvc, update_shard_cc)) {
auto csr = shard_conf_->get_root_csr();
if (csr.is_null()) {
LOG(WARNING) << "new shard configuration is null (!)";
} else {
LOG(WARNING) << "invalid new shard configuration is";
std::ostringstream os;
csr->print_rec(os);
block::gen::t_ShardHashes.print(os, csr.write());
LOG(WARNING) << os.str();
FLOG(WARNING) {
csr->print_rec(sb);
block::gen::t_ShardHashes.print(sb, csr);
};
}
return fatal_error("cannot post-process shard configuration");
}
// 3. save new shard_hashes
state_extra.shard_hashes = shard_conf_->get_root_csr();
if (verbosity >= 3) {
std::cerr << "updated shard configuration to ";
block::gen::t_ShardHashes.print(std::cerr, *state_extra.shard_hashes);
FLOG(INFO) {
sb << "updated shard configuration to ";
block::gen::t_ShardHashes.print(sb, state_extra.shard_hashes);
};
}
if (!block::gen::t_ShardHashes.validate_upto(10000, *state_extra.shard_hashes)) {
return fatal_error("new ShardHashes is invalid");
@ -4777,13 +4808,18 @@ bool Collator::create_mc_state_extra() {
if (verify >= 2) {
LOG(INFO) << "verifying new BlockCreateStats";
if (!block::gen::t_BlockCreateStats.validate_csr(100000, cs)) {
cs->print_rec(std::cerr);
block::gen::t_BlockCreateStats.print(std::cerr, *cs);
FLOG(WARNING) {
sb << "BlockCreateStats in the new masterchain state failed to pass automated validity checks: ";
cs->print_rec(sb);
block::gen::t_BlockCreateStats.print(sb, cs);
};
return fatal_error("BlockCreateStats in the new masterchain state failed to pass automated validity checks");
}
}
if (verbosity >= 4 * 1) {
block::gen::t_BlockCreateStats.print(std::cerr, *cs);
FLOG(INFO) {
block::gen::t_BlockCreateStats.print(sb, cs);
};
}
} else {
state_extra.r1.block_create_stats.clear();
@ -4818,7 +4854,6 @@ bool Collator::update_block_creator_count(td::ConstBitPtr key, unsigned shard_in
if (!block::unpack_CreatorStats(std::move(cs), mc_cnt, shard_cnt)) {
return fatal_error("cannot unpack CreatorStats for "s + key.to_hex(256) + " from previous masterchain state");
}
// std::cerr << mc_cnt.to_str() << " " << shard_cnt.to_str() << std::endl;
if (mc_incr && !mc_cnt.increase_by(mc_incr, now_)) {
return fatal_error(PSTRING() << "cannot increase masterchain block counter in CreatorStats for " << key.to_hex(256)
<< " by " << mc_incr << " (old value is " << mc_cnt.to_str() << ")");
@ -5190,9 +5225,11 @@ bool Collator::update_public_libraries() {
}
}
if (libraries_changed_ && verbosity >= 2) {
std::cerr << "New public libraries: ";
block::gen::t_HashmapE_256_LibDescr.print(std::cerr, shard_libraries_->get_root());
shard_libraries_->get_root()->print_rec(std::cerr);
FLOG(INFO) {
sb << "New public libraries: ";
block::gen::t_HashmapE_256_LibDescr.print(sb, shard_libraries_->get_root());
shard_libraries_->get_root()->print_rec(sb);
};
}
return true;
}
@ -5315,9 +5352,11 @@ bool Collator::create_shard_state() {
}
LOG(DEBUG) << "min_ref_mc_seqno is " << min_ref_mc_seqno_;
if (verbosity > 2) {
std::cerr << "new ShardState: ";
block::gen::t_ShardState.print_ref(std::cerr, state_root);
vm::load_cell_slice(state_root).print_rec(std::cerr);
FLOG(INFO) {
sb << "new ShardState: ";
block::gen::t_ShardState.print_ref(sb, state_root);
vm::load_cell_slice(state_root).print_rec(sb);
};
}
if (verify >= 2) {
LOG(INFO) << "verifying new ShardState";
@ -5330,9 +5369,11 @@ bool Collator::create_shard_state() {
return fatal_error("cannot create Merkle update for ShardState");
}
if (verbosity > 2) {
std::cerr << "Merkle Update for ShardState: ";
vm::CellSlice cs{vm::NoVm{}, state_update};
cs.print_rec(std::cerr);
FLOG(INFO) {
sb << "Merkle Update for ShardState: ";
vm::CellSlice cs{vm::NoVm{}, state_update};
cs.print_rec(sb);
};
}
LOG(INFO) << "updating block profile statistics";
block_limit_status_->add_proof(state_root);
@ -5377,10 +5418,12 @@ bool Collator::update_processed_upto() {
*/
bool Collator::compute_out_msg_queue_info(Ref<vm::Cell>& out_msg_queue_info) {
if (verbosity >= 2) {
auto rt = out_msg_queue_->get_root();
std::cerr << "resulting out_msg_queue is ";
block::gen::t_OutMsgQueue.print(std::cerr, *rt);
rt->print_rec(std::cerr);
FLOG(INFO) {
auto rt = out_msg_queue_->get_root();
sb << "resulting out_msg_queue is ";
block::gen::t_OutMsgQueue.print(sb, rt);
rt->print_rec(sb);
};
}
vm::CellBuilder cb;
// out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) = OutMsgQueueExtra;
@ -5430,8 +5473,10 @@ bool Collator::compute_total_balance() {
}
vm::CellSlice cs{*(in_msg_dict->get_root_extra())};
if (verbosity > 2) {
block::gen::t_ImportFees.print(std::cerr, vm::CellSlice{*(in_msg_dict->get_root_extra())});
cs.print_rec(std::cerr);
FLOG(INFO) {
block::gen::t_ImportFees.print(sb, in_msg_dict->get_root_extra());
cs.print_rec(sb);
};
}
auto new_import_fees = block::tlb::t_Grams.as_integer_skip(cs);
if (new_import_fees.is_null()) {
@ -5659,9 +5704,11 @@ bool Collator::create_block() {
return fatal_error("cannot create new Block");
}
if (verbosity >= 3 * 1) {
std::cerr << "new Block: ";
block::gen::t_Block.print_ref(std::cerr, new_block);
vm::load_cell_slice(new_block).print_rec(std::cerr);
FLOG(INFO) {
sb << "new Block: ";
block::gen::t_Block.print_ref(sb, new_block);
vm::load_cell_slice(new_block).print_rec(sb);
};
}
if (verify >= 1) {
LOG(INFO) << "verifying new Block";
@ -5699,9 +5746,11 @@ Ref<vm::Cell> Collator::collate_shard_block_descr_set() {
return {};
}
if (verbosity >= 4 * 1) {
std::cerr << "serialized TopBlockDescrSet for collated data is: ";
block::gen::t_TopBlockDescrSet.print_ref(std::cerr, cell);
vm::load_cell_slice(cell).print_rec(std::cerr);
FLOG(INFO) {
sb << "serialized TopBlockDescrSet for collated data is: ";
block::gen::t_TopBlockDescrSet.print_ref(sb, cell);
vm::load_cell_slice(cell).print_rec(sb);
};
}
return cell;
}
@ -5965,35 +6014,6 @@ bool Collator::create_block_candidate() {
td::actor::send_closure_later(manager, &ValidatorManager::complete_external_messages, std::move(delay_ext_msgs_),
std::move(bad_ext_msgs_));
}
double work_time = work_timer_.elapsed();
double cpu_work_time = cpu_work_timer_.elapsed();
LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s";
stats_.block_id = block_candidate->id;
stats_.collated_data_hash = block_candidate->collated_file_hash;
stats_.cc_seqno = validator_set_->get_catchain_seqno();
stats_.collated_at = td::Clocks::system();
stats_.actual_bytes = block_candidate->data.size();
stats_.actual_collated_data_bytes = block_candidate->collated_data.size();
stats_.attempt = attempt_idx_;
stats_.is_validator = !(mode_ & CollateMode::from_collator_node);
stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id()
: collator_node_id_.pubkey_hash();
stats_.estimated_bytes = block_limit_status_->estimate_block_size();
stats_.gas = block_limit_status_->gas_used;
stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt;
stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_stat.estimate_proof_size();
stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes);
stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas);
stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt);
stats_.cat_collated_data_bytes =
block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes);
stats_.total_time = perf_timer_.elapsed();
stats_.work_time = work_time;
stats_.cpu_work_time = cpu_work_time;
stats_.time_stats = (PSTRING() << perf_log_);
td::actor::send_closure(manager, &ValidatorManager::log_collate_query_stats, std::move(stats_));
return true;
}
@ -6013,6 +6033,9 @@ void Collator::return_block_candidate(td::Result<td::Unit> saved) {
LOG(WARNING) << "sending new BlockCandidate to Promise";
LOG(WARNING) << "collation took " << perf_timer_.elapsed() << " s";
LOG(WARNING) << perf_log_;
finalize_stats();
stats_.status = td::Status::OK();
td::actor::send_closure(manager, &ValidatorManager::log_collate_query_stats, std::move(stats_));
main_promise(block_candidate->clone());
busy_ = false;
stop();
@ -6078,8 +6101,10 @@ td::Result<bool> Collator::register_external_message_cell(Ref<vm::Cell> ext_msg,
return td::Status::Error("inbound external message has destination address not in this shard");
}
if (verbosity > 2) {
std::cerr << "registered external message: ";
block::gen::t_Message_Any.print_ref(std::cerr, ext_msg);
FLOG(INFO) {
sb << "registered external message: ";
block::gen::t_Message_Any.print_ref(sb, ext_msg);
};
}
ext_msg_map.emplace(hash, 1);
ext_msg_list_.push_back({std::move(ext_msg), ext_hash, priority});
@ -6139,6 +6164,39 @@ td::uint32 Collator::get_skip_externals_queue_size() {
return SKIP_EXTERNALS_QUEUE_SIZE;
}
void Collator::finalize_stats() {
double work_time = work_timer_.elapsed();
double cpu_work_time = cpu_work_timer_.elapsed();
LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s";
if (block_candidate) {
stats_.block_id = block_candidate->id;
stats_.collated_data_hash = block_candidate->collated_file_hash;
stats_.actual_bytes = block_candidate->data.size();
stats_.actual_collated_data_bytes = block_candidate->collated_data.size();
} else {
stats_.block_id.id = new_id;
}
stats_.cc_seqno = validator_set_->get_catchain_seqno();
stats_.collated_at = td::Clocks::system();
stats_.attempt = attempt_idx_;
stats_.is_validator = !(mode_ & CollateMode::from_collator_node);
stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id()
: collator_node_id_.pubkey_hash();
stats_.estimated_bytes = block_limit_status_->estimate_block_size();
stats_.gas = block_limit_status_->gas_used;
stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt;
stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_stat.estimate_proof_size();
stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes);
stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas);
stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt);
stats_.cat_collated_data_bytes =
block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes);
stats_.total_time = perf_timer_.elapsed();
stats_.work_time = work_time;
stats_.cpu_work_time = cpu_work_time;
stats_.time_stats = (PSTRING() << perf_log_);
}
} // namespace validator
} // namespace ton

17
validator/impl/liteserver.cpp
View file

@ -85,19 +85,13 @@ void LiteQuery::abort_query(td::Status reason) {
if (acc_state_promise_) {
acc_state_promise_.set_error(std::move(reason));
} else if (promise_) {
td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_ ? query_obj_->get_id() : 0,
false);
promise_.set_error(std::move(reason));
}
stop();
}
void LiteQuery::abort_query_ext(td::Status reason, std::string comment) {
LOG(INFO) << "aborted liteserver query: " << comment << " : " << reason.to_string();
if (promise_) {
promise_.set_error(reason.move_as_error_prefix(comment + " : "));
}
stop();
}
bool LiteQuery::fatal_error(td::Status error) {
abort_query(std::move(error));
return false;
@ -120,6 +114,8 @@ bool LiteQuery::finish_query(td::BufferSlice result, bool skip_cache_update) {
td::actor::send_closure(cache_, &LiteServerCache::update, cache_key_, result.clone());
}
if (promise_) {
td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_ ? query_obj_->get_id() : 0,
true);
promise_.set_result(std::move(result));
stop();
return true;
@ -139,7 +135,6 @@ void LiteQuery::start_up() {
auto F = fetch_tl_object<ton::lite_api::Function>(query_, true);
if (F.is_error()) {
td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, 0); // unknown
abort_query(F.move_as_error());
return;
}
@ -192,7 +187,6 @@ bool LiteQuery::use_cache() {
}
void LiteQuery::perform() {
td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_->get_id());
lite_api::downcast_call(
*query_obj_,
td::overloaded(
@ -1905,6 +1899,9 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector<int> param_list)
}
cfg = res.move_as_ok();
} else {
if (mode & block::ConfigInfo::needPrevBlocks) {
mode |= block::ConfigInfo::needCapabilities;
}
auto res = block::ConfigInfo::extract_config(mpb.root(), mode);
if (res.is_error()) {
fatal_error(res.move_as_error());

1
validator/impl/liteserver.hpp
View file

@ -97,7 +97,6 @@ class LiteQuery : public td::actor::Actor {
bool fatal_error(std::string err_msg, int err_code = -400);
bool fatal_error(int err_code, std::string err_msg = "");
void abort_query(td::Status reason);
void abort_query_ext(td::Status reason, std::string err_msg);
bool finish_query(td::BufferSlice result, bool skip_cache_update = false);
void alarm() override;
void start_up() override;

3
validator/impl/signature-set.cpp
View file

@ -42,9 +42,6 @@ td::BufferSlice BlockSignatureSetQ::serialize() const {
}
Ref<vm::Cell> root;
CHECK(serialize_to(root));
//std::cerr << "serializing BlockSignatureSet: ";
//vm::CellSlice{vm::NoVm{}, root}.print_rec(std::cerr);
//std::cerr << std::endl;
auto res = vm::std_boc_serialize(std::move(root));
LOG_CHECK(res.is_ok()) << res.move_as_error();
return res.move_as_ok();

8
validator/impl/top-shard-descr.cpp
View file

@ -175,9 +175,11 @@ td::Status ShardTopBlockDescrQ::unpack() {
block::gen::TopBlockDescr::Record rec;
if (!(block::gen::t_TopBlockDescr.force_validate_ref(root_) && tlb::unpack_cell(root_, rec) &&
block::tlb::t_BlockIdExt.unpack(rec.proof_for.write(), block_id_))) {
std::cerr << "invalid ShardTopBlockDescr: ";
block::gen::t_TopBlockDescr.print_ref(std::cerr, root_);
vm::load_cell_slice(root_).print_rec(std::cerr);
FLOG(INFO) {
sb << "invalid ShardTopBlockDescr: ";
block::gen::t_TopBlockDescr.print_ref(sb, root_);
vm::load_cell_slice(root_).print_rec(sb);
};
return td::Status::Error(-666, "Shard top block description is not a valid TopBlockDescr TL-B object");
}
LOG(DEBUG) << "unpacking a ShardTopBlockDescr for " << block_id_.to_str() << " with " << rec.len << " links";

71
validator/impl/validate-query.cpp
View file

@ -1613,8 +1613,10 @@ void ValidateQuery::got_neighbor_out_queue(int i, td::Result<Ref<MessageQueue>>
// unpack ProcessedUpto
LOG(DEBUG) << "unpacking ProcessedUpto of neighbor " << descr.blk_.to_str();
if (verbosity >= 2) {
block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info);
qinfo.proc_info->print_rec(std::cerr);
FLOG(INFO) {
block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info);
qinfo.proc_info->print_rec(sb);
};
}
descr.processed_upto = block::MsgProcessedUptoCollection::unpack(descr.shard(), qinfo.proc_info);
if (!descr.processed_upto) {
@ -2728,7 +2730,6 @@ bool ValidateQuery::unpack_precheck_value_flow(Ref<vm::Cell> value_flow_root) {
" but the sum over all accounts present in the new state is " + cc.to_str());
}
auto msg_extra = in_msg_dict_->get_root_extra();
// block::gen::t_ImportFees.print(std::cerr, msg_extra);
if (!(block::tlb::t_Grams.as_integer_skip_to(msg_extra.write(), import_fees_) && cc.unpack(std::move(msg_extra)))) {
return reject_query("cannot unpack ImportFees from the augmentation of the InMsgDescr dictionary");
}
@ -2832,20 +2833,22 @@ bool ValidateQuery::precheck_one_account_update(td::ConstBitPtr acc_id, Ref<vm::
auto acc_blk_root = account_blocks_dict_->lookup(acc_id, 256);
if (acc_blk_root.is_null()) {
if (verbosity >= 3 * 0) {
std::cerr << "state of account " << workchain() << ":" << acc_id.to_hex(256)
<< " in the old shardchain state:" << std::endl;
if (old_value.not_null()) {
block::gen::t_ShardAccount.print(std::cerr, *old_value);
} else {
std::cerr << "<absent>" << std::endl;
}
std::cerr << "state of account " << workchain() << ":" << acc_id.to_hex(256)
<< " in the new shardchain state:" << std::endl;
if (new_value.not_null()) {
block::gen::t_ShardAccount.print(std::cerr, *new_value);
} else {
std::cerr << "<absent>" << std::endl;
}
FLOG(INFO) {
sb << "state of account " << workchain() << ":" << acc_id.to_hex(256)
<< " in the old shardchain state:" << "\n";
if (old_value.not_null()) {
block::gen::t_ShardAccount.print(sb, old_value);
} else {
sb << "<absent>" << "\n";
}
sb << "state of account " << workchain() << ":" << acc_id.to_hex(256)
<< " in the new shardchain state:" << "\n";
if (new_value.not_null()) {
block::gen::t_ShardAccount.print(sb, new_value);
} else {
sb << "<absent>" << "\n";
}
};
}
return reject_query("the state of account "s + acc_id.to_hex(256) +
" changed in the new state with respect to the old state, but the block contains no "
@ -3003,8 +3006,6 @@ bool ValidateQuery::precheck_one_account_block(td::ConstBitPtr acc_id, Ref<vm::C
" not belonging to the block's shard " + shard_.to_str());
}
CHECK(acc_blk_root.not_null());
// acc_blk_root->print_rec(std::cerr);
// block::gen::t_AccountBlock.print(std::cerr, acc_blk_root);
block::gen::AccountBlock::Record acc_blk;
block::gen::HASH_UPDATE::Record hash_upd;
if (!(tlb::csr_unpack(acc_blk_root, acc_blk) &&
@ -3935,7 +3936,9 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref<vm::CellSlice> in_msg)
ton::LogicalTime trans_lt;
CHECK(block::get_transaction_id(transaction, trans_addr, trans_lt));
if (dest_addr != trans_addr) {
block::gen::t_InMsg.print(std::cerr, *in_msg);
FLOG(INFO) {
block::gen::t_InMsg.print(sb, in_msg);
};
return reject_query(PSTRING() << "InMsg corresponding to inbound message with hash " << key.to_hex(256)
<< " and destination address " << dest_addr.to_hex()
<< " claims that the message is processed by transaction " << trans_lt
@ -4483,7 +4486,9 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref<vm::CellSlice> out_ms
ton::LogicalTime trans_lt;
CHECK(block::get_transaction_id(transaction, trans_addr, trans_lt));
if (src_addr != trans_addr) {
block::gen::t_OutMsg.print(std::cerr, *out_msg);
FLOG(INFO) {
block::gen::t_OutMsg.print(sb, out_msg);
};
return reject_query(PSTRING() << "OutMsg corresponding to outbound message with hash " << key.to_hex(256)
<< " and source address " << src_addr.to_hex()
<< " claims that the message was created by transaction " << trans_lt
@ -5111,8 +5116,10 @@ bool ValidateQuery::check_in_queue() {
LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex()
<< ") from neighbor #" << kv->source;
if (verbosity > 3) {
std::cerr << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg=";
block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg));
FLOG(INFO) {
sb << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg=";
block::gen::t_EnqueuedMsg.print(sb, kv->msg);
};
}
bool unprocessed = false;
bool processed_here = false;
@ -5120,9 +5127,11 @@ bool ValidateQuery::check_in_queue() {
if (!check_neighbor_outbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source), unprocessed,
processed_here, msg_hash)) {
if (verbosity > 1) {
std::cerr << "invalid neighbor outbound message: lt=" << kv->lt << " from=" << kv->source
<< " key=" << kv->key.to_hex() << " msg=";
block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg));
FLOG(INFO) {
sb << "invalid neighbor outbound message: lt=" << kv->lt << " from=" << kv->source
<< " key=" << kv->key.to_hex() << " msg=";
block::gen::t_EnqueuedMsg.print(sb, kv->msg);
};
}
return reject_query("error processing outbound internal message "s + kv->key.to_hex() + " of neighbor " +
neighbors_.at(kv->source).blk_.to_str());
@ -5735,10 +5744,12 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT
// now compare the re-created transaction with the one we have
if (trans_root2->get_hash() != trans_root->get_hash()) {
if (verbosity >= 3 * 0) {
std::cerr << "original transaction " << lt << " of " << addr.to_hex() << ": ";
block::gen::t_Transaction.print_ref(std::cerr, trans_root);
std::cerr << "re-created transaction " << lt << " of " << addr.to_hex() << ": ";
block::gen::t_Transaction.print_ref(std::cerr, trans_root2);
FLOG(INFO) {
sb << "original transaction " << lt << " of " << addr.to_hex() << ": ";
block::gen::t_Transaction.print_ref(sb, trans_root);
sb << "re-created transaction " << lt << " of " << addr.to_hex() << ": ";
block::gen::t_Transaction.print_ref(sb, trans_root2);
};
}
return reject_query(PSTRING() << "the transaction " << lt << " of " << addr.to_hex() << " has hash "
<< trans_root->get_hash().to_hex()

6
validator/interfaces/validator-manager.h
View file

@ -56,7 +56,9 @@ struct AsyncSerializerState {
};
struct CollationStats {
BlockIdExt block_id;
BlockIdExt block_id{workchainInvalid, 0, 0, RootHash::zero(), FileHash::zero()};
td::Status status = td::Status::OK();
td::Bits256 collated_data_hash = td::Bits256::zero();
CatchainSeqno cc_seqno = 0;
double collated_at = -1.0;
@ -265,7 +267,7 @@ class ValidatorManager : public ValidatorManagerInterface {
td::optional<ShardIdFull> shard,
td::Promise<tl_object_ptr<lite_api::liteServer_nonfinal_validatorGroups>> promise) = 0;
virtual void add_lite_query_stats(int lite_query_id) {
virtual void add_lite_query_stats(int lite_query_id, bool success) {
}
virtual void log_collate_query_stats(CollationStats stats) {

6
validator/manager-init.cpp
View file

@ -32,6 +32,8 @@ namespace ton {
namespace validator {
void ValidatorManagerMasterchainReiniter::start_up() {
status_ = ProcessStatus(manager_, "process.initial_sync");
status_.set_status(PSTRING() << "starting, init block seqno " << block_id_.seqno());
LOG(INFO) << "init_block_id=" << block_id_;
CHECK(block_id_.is_masterchain());
CHECK(block_id_.id.shard == shardIdAll);
@ -58,6 +60,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han
key_blocks_.push_back(handle_);
if (opts_->initial_sync_disabled()) {
status_.set_status(PSTRING() << "downloading masterchain state " << handle_->id().seqno());
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
R.ensure();
td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_masterchain_state);
@ -181,6 +184,7 @@ void ValidatorManagerMasterchainReiniter::got_next_key_blocks(std::vector<BlockI
}
}
LOG(WARNING) << "last key block is " << vec[vec.size() - 1];
status_.set_status(PSTRING() << "last key block is " << vec.back().seqno());
auto s = static_cast<td::uint32>(key_blocks_.size());
key_blocks_.resize(key_blocks_.size() + vec.size(), nullptr);
@ -247,6 +251,7 @@ void ValidatorManagerMasterchainReiniter::choose_masterchain_state() {
}
void ValidatorManagerMasterchainReiniter::download_masterchain_state() {
status_.set_status(PSTRING() << "downloading masterchain state " << block_id_.seqno());
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
if (R.is_error()) {
LOG(WARNING) << "failed to download masterchain state: " << R.move_as_error();
@ -274,6 +279,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Ref<S
td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_all_shards);
});
client_ = td::actor::create_actor<ShardClient>("shardclient", opts_, handle_, state_, manager_, std::move(P));
status_.set_status(PSTRING() << "downloading all shard states, mc seqno " << block_id_.seqno());
}
void ValidatorManagerMasterchainReiniter::downloaded_all_shards() {

4
validator/manager-init.hpp
View file

@ -27,6 +27,8 @@
#include "manager-init.h"
#include <stats-provider.h>
namespace ton {
namespace validator {
@ -77,6 +79,8 @@ class ValidatorManagerMasterchainReiniter : public td::actor::Actor {
td::uint32 pending_ = 0;
td::actor::ActorOwn<ShardClient> client_;
ProcessStatus status_;
};
class ValidatorManagerMasterchainStarter : public td::actor::Actor {

106
validator/manager.cpp
View file

@ -437,6 +437,10 @@ void ValidatorManagerImpl::add_external_message(td::Ref<ExtMessage> msg, int pri
ext_messages_hashes_[id.hash] = {priority, id};
}
void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise<td::Ref<ExtMessage>> promise) {
if (!started_) {
promise.set_error(td::Status::Error(ErrorCode::notready, "node not synced"));
return;
}
auto state = do_get_last_liteserver_state();
if (state.is_null()) {
promise.set_error(td::Status::Error(ErrorCode::notready, "not ready"));
@ -458,11 +462,9 @@ void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Prom
promise = [self = this, wc, addr, promise = std::move(promise),
SelfId = actor_id(this)](td::Result<td::Ref<ExtMessage>> R) mutable {
if (R.is_error()) {
promise.set_error(R.move_as_error());
return;
}
td::actor::send_lambda(SelfId, [=, promise = std::move(promise), message = R.move_as_ok()]() mutable {
td::actor::send_lambda(SelfId, [=, promise = std::move(promise), R = std::move(R)]() mutable {
++(R.is_ok() ? self->total_check_ext_messages_ok_ : self->total_check_ext_messages_error_);
TRY_RESULT_PROMISE(promise, message, std::move(R));
if (self->checked_ext_msg_counter_.inc_msg_count(wc, addr) > max_ext_msg_per_addr()) {
promise.set_error(
td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex()));
@ -760,11 +762,10 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Ref<ShardState>> R) {
td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R));
});
auto id =
td::actor::create_actor<WaitBlockState>("waitstate", handle, priority, actor_id(this),
td::Timestamp::at(timeout.at() + 10.0), std::move(P),
get_block_persistent_state(handle->id()))
.release();
auto id = td::actor::create_actor<WaitBlockState>("waitstate", handle, priority, actor_id(this),
td::Timestamp::at(timeout.at() + 10.0), std::move(P),
get_block_persistent_state_to_download(handle->id()))
.release();
wait_state_[handle->id()].actor_ = id;
it = wait_state_.find(handle->id());
}
@ -1283,9 +1284,10 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result<td
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Ref<ShardState>> R) {
td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R));
});
auto id = td::actor::create_actor<WaitBlockState>("waitstate", handle, X.second, actor_id(this), X.first,
std::move(P), get_block_persistent_state(handle->id()))
.release();
auto id =
td::actor::create_actor<WaitBlockState>("waitstate", handle, X.second, actor_id(this), X.first,
std::move(P), get_block_persistent_state_to_download(handle->id()))
.release();
it->second.actor_ = id;
return;
}
@ -2290,7 +2292,7 @@ void ValidatorManagerImpl::update_shards() {
}
}
bool validating_masterchain = false;
active_validator_groups_master_ = active_validator_groups_shard_ = 0;
if (allow_validate_) {
for (auto &desc : new_shards) {
auto shard = desc.first;
@ -2307,9 +2309,7 @@ void ValidatorManagerImpl::update_shards() {
auto validator_id = get_validator(shard, val_set);
if (!validator_id.is_zero()) {
if (shard.is_masterchain()) {
validating_masterchain = true;
}
++(shard.is_masterchain() ? active_validator_groups_master_ : active_validator_groups_shard_);
auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts);
if (force_recover) {
@ -3044,8 +3044,8 @@ void ValidatorManagerImpl::prepare_stats(td::Promise<std::vector<std::pair<std::
vec.emplace_back("knownkeymasterchainblock", last_known_key_block_handle_->id().to_str());
vec.emplace_back("rotatemasterchainblock", last_rotate_block_id_.to_str());
//vec.emplace_back("shardclientmasterchainseqno", td::to_string(min_confirmed_masterchain_seqno_));
vec.emplace_back("stateserializermasterchainseqno", td::to_string(state_serializer_masterchain_seqno_));
}
td::NamedThreadSafeCounter::get_default().for_each([&](auto key, auto value) {
vec.emplace_back("counter." + key, PSTRING() << value);
});
@ -3063,9 +3063,48 @@ void ValidatorManagerImpl::prepare_stats(td::Promise<std::vector<std::pair<std::
td::actor::send_closure(shard_client_, &ShardClient::get_processed_masterchain_block, std::move(P));
}
vec.emplace_back("start_time", td::to_string(started_at_));
for (int iter = 0; iter < 2; ++iter) {
td::StringBuilder sb;
td::uint32 total = 0;
for (const auto &p : (iter ? total_ls_queries_error_ : total_ls_queries_ok_)) {
sb << lite_query_name_by_id(p.first) << ":" << p.second << " ";
total += p.second;
}
sb << "TOTAL:" << total;
vec.emplace_back(PSTRING() << "total.ls_queries_" << (iter ? "error" : "ok"), sb.as_cslice().str());
}
vec.emplace_back("total.ext_msg_check",
PSTRING() << "ok:" << total_check_ext_messages_ok_ << " error:" << total_check_ext_messages_error_);
vec.emplace_back("total.collated_blocks.master", PSTRING() << "ok:" << total_collated_blocks_master_ok_
<< " error:" << total_collated_blocks_master_error_);
vec.emplace_back("total.collated_blocks.shard", PSTRING() << "ok:" << total_collated_blocks_shard_ok_
<< " error:" << total_collated_blocks_shard_error_);
vec.emplace_back("total.validated_blocks.master", PSTRING() << "ok:" << total_validated_blocks_master_ok_
<< " error:" << total_validated_blocks_master_error_);
vec.emplace_back("total.validated_blocks.shard", PSTRING() << "ok:" << total_validated_blocks_shard_ok_
<< " error:" << total_validated_blocks_shard_error_);
if (is_validator()) {
vec.emplace_back("active_validator_groups", PSTRING() << "master:" << active_validator_groups_master_
<< " shard:" << active_validator_groups_shard_);
}
bool serializer_enabled = opts_->get_state_serializer_enabled();
if (is_validator() && last_masterchain_state_->get_global_id() == -239) {
serializer_enabled = false;
}
vec.emplace_back("stateserializerenabled", serializer_enabled ? "true" : "false");
merger.make_promise("").set_value(std::move(vec));
if (!serializer_.empty()) {
td::actor::send_closure(serializer_, &AsyncStateSerializer::prepare_stats, merger.make_promise(""));
}
td::actor::send_closure(db_, &Db::prepare_stats, merger.make_promise("db."));
for (auto &[_, p] : stats_providers_) {
p.second(merger.make_promise(p.first));
}
}
void ValidatorManagerImpl::prepare_perf_timer_stats(td::Promise<std::vector<PerfTimerStats>> promise) {
@ -3551,11 +3590,18 @@ void ValidatorManagerImpl::got_persistent_state_descriptions(std::vector<td::Ref
}
}
td::Ref<PersistentStateDescription> ValidatorManagerImpl::get_block_persistent_state(BlockIdExt block_id) {
td::Ref<PersistentStateDescription> ValidatorManagerImpl::get_block_persistent_state_to_download(BlockIdExt block_id) {
if (block_id.is_masterchain()) {
return {};
}
auto it = persistent_state_blocks_.find(block_id);
if (it == persistent_state_blocks_.end()) {
return {};
}
if (it->second->masterchain_id.seqno() + 16 >= min_confirmed_masterchain_seqno_) {
// Do not download persistent states during ordinary shard client sync
return {};
}
return it->second;
}
@ -3568,10 +3614,20 @@ td::actor::ActorOwn<ValidatorManagerInterface> ValidatorManagerFactory::create(
}
void ValidatorManagerImpl::log_collate_query_stats(CollationStats stats) {
write_session_stats(stats);
if (stats.status.is_ok()) {
++(stats.block_id.is_masterchain() ? total_collated_blocks_master_ok_ : total_collated_blocks_shard_ok_);
write_session_stats(stats);
} else {
++(stats.block_id.is_masterchain() ? total_collated_blocks_master_error_ : total_collated_blocks_shard_error_);
}
}
void ValidatorManagerImpl::log_validate_query_stats(ValidationStats stats) {
if (stats.valid) {
++(stats.block_id.is_masterchain() ? total_validated_blocks_master_ok_ : total_validated_blocks_shard_ok_);
} else {
++(stats.block_id.is_masterchain() ? total_validated_blocks_master_error_ : total_validated_blocks_shard_error_);
}
write_session_stats(stats);
}
@ -3579,6 +3635,16 @@ void ValidatorManagerImpl::log_collator_node_response_stats(CollatorNodeResponse
write_session_stats(stats);
}
void ValidatorManagerImpl::register_stats_provider(
td::uint64 idx, std::string prefix,
std::function<void(td::Promise<std::vector<std::pair<std::string, std::string>>>)> callback) {
stats_providers_[idx] = {std::move(prefix), std::move(callback)};
}
void ValidatorManagerImpl::unregister_stats_provider(td::uint64 idx) {
stats_providers_.erase(idx);
}
size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) {
before_query();
auto it1 = counter_cur_.find({wc, addr});

24
validator/manager.hpp
View file

@ -681,8 +681,9 @@ class ValidatorManagerImpl : public ValidatorManager {
td::optional<ShardIdFull> shard,
td::Promise<tl_object_ptr<lite_api::liteServer_nonfinal_validatorGroups>> promise) override;
void add_lite_query_stats(int lite_query_id) override {
void add_lite_query_stats(int lite_query_id, bool success) override {
++ls_stats_[lite_query_id];
++(success ? total_ls_queries_ok_ : total_ls_queries_error_)[lite_query_id];
}
private:
@ -759,7 +760,7 @@ class ValidatorManagerImpl : public ValidatorManager {
void got_persistent_state_descriptions(std::vector<td::Ref<PersistentStateDescription>> descs);
void add_persistent_state_description_impl(td::Ref<PersistentStateDescription> desc);
td::Ref<PersistentStateDescription> get_block_persistent_state(BlockIdExt block_id);
td::Ref<PersistentStateDescription> get_block_persistent_state_to_download(BlockIdExt block_id);
private:
bool need_monitor(ShardIdFull shard) const {
@ -773,12 +774,27 @@ class ValidatorManagerImpl : public ValidatorManager {
std::map<int, td::uint32> ls_stats_; // lite_api ID -> count, 0 for unknown
td::uint32 ls_stats_check_ext_messages_{0};
UnixTime started_at_ = (UnixTime)td::Clocks::system();
std::map<int, td::uint64> total_ls_queries_ok_, total_ls_queries_error_; // lite_api ID -> count, 0 for unknown
td::uint64 total_check_ext_messages_ok_{0}, total_check_ext_messages_error_{0};
td::uint64 total_collated_blocks_master_ok_{0}, total_collated_blocks_master_error_{0};
td::uint64 total_validated_blocks_master_ok_{0}, total_validated_blocks_master_error_{0};
td::uint64 total_collated_blocks_shard_ok_{0}, total_collated_blocks_shard_error_{0};
td::uint64 total_validated_blocks_shard_ok_{0}, total_validated_blocks_shard_error_{0};
size_t active_validator_groups_master_{0}, active_validator_groups_shard_{0};
td::actor::ActorOwn<CandidatesBuffer> candidates_buffer_;
void log_collate_query_stats(CollationStats stats) override;
void log_validate_query_stats(ValidationStats stats) override;
void log_collator_node_response_stats(CollatorNodeResponseStats stats) override;
void register_stats_provider(
td::uint64 idx, std::string prefix,
std::function<void(td::Promise<std::vector<std::pair<std::string, std::string>>>)> callback) override;
void unregister_stats_provider(td::uint64 idx) override;
std::map<PublicKeyHash, td::actor::ActorOwn<ValidatorTelemetry>> validator_telemetry_;
void init_validator_telemetry();
@ -794,6 +810,10 @@ class ValidatorManagerImpl : public ValidatorManager {
std::map<BlockSeqno, td::Ref<PersistentStateDescription>> persistent_state_descriptions_;
std::map<BlockIdExt, td::Ref<PersistentStateDescription>> persistent_state_blocks_;
std::map<td::uint64,
std::pair<std::string, std::function<void(td::Promise<std::vector<std::pair<std::string, std::string>>>)>>>
stats_providers_;
bool session_stats_enabled_ = false;
std::string session_stats_filename_;
td::FileFd session_stats_fd_;

10
validator/net/download-state.cpp
View file

@ -70,6 +70,7 @@ void DownloadState::finish_query() {
}
void DownloadState::start_up() {
status_ = ProcessStatus(validator_manager_, "process.download_state_net");
alarm_timestamp() = timeout_;
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_,
@ -190,6 +191,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) {
td::Timestamp::in(3.0), std::move(P));
}
}));
status_.set_status(PSTRING() << block_id_.id.to_str() << " : 0 bytes, 0B/s");
}
void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 requested_size) {
@ -198,14 +200,18 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques
parts_.push_back(std::move(data));
double elapsed = prev_logged_timer_.elapsed();
if (elapsed > 10.0) {
if (elapsed > 5.0) {
prev_logged_timer_ = td::Timer();
auto speed = (td::uint64)((double)(sum_ - prev_logged_sum_) / elapsed);
LOG(WARNING) << "downloading state " << block_id_.to_str() << ": " << td::format::as_size(sum_) << " ("
<< td::format::as_size((td::uint64)(double(sum_ - prev_logged_sum_) / elapsed)) << "/s)";
<< td::format::as_size(speed) << "/s)";
status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, " << td::format::as_size(speed)
<< "/s");
prev_logged_sum_ = sum_;
}
if (last_part) {
status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, finishing");
td::BufferSlice res{td::narrow_cast<std::size_t>(sum_)};
auto S = res.as_slice();
for (auto &p : parts_) {

4
validator/net/download-state.hpp
View file

@ -23,6 +23,8 @@
#include "validator/validator.h"
#include "adnl/adnl-ext-client.h"
#include <stats-provider.h>
namespace ton {
namespace validator {
@ -75,6 +77,8 @@ class DownloadState : public td::actor::Actor {
td::uint64 prev_logged_sum_ = 0;
td::Timer prev_logged_timer_;
ProcessStatus status_;
};
} // namespace fullnode

44
validator/state-serializer.cpp
View file

@ -58,6 +58,12 @@ void AsyncStateSerializer::got_self_state(AsyncSerializerState state) {
});
td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, last_block_id_, true, std::move(P));
}
inited_block_id_ = true;
for (auto& promise : wait_init_block_id_) {
promise.set_value(td::Unit());
}
wait_init_block_id_.clear();
}
void AsyncStateSerializer::got_init_handle(BlockHandle handle) {
@ -186,6 +192,9 @@ void AsyncStateSerializer::next_iteration() {
td::actor::send_closure(SelfId, &AsyncStateSerializer::request_previous_state_files);
},
td::Timestamp::in(delay));
current_status_ = PSTRING() << "delay before serializing seqno=" << masterchain_handle_->id().seqno() << " "
<< (int)delay << "s";
current_status_ts_ = td::Timestamp::now();
return;
}
if (next_idx_ < shards_.size()) {
@ -379,9 +388,14 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state
td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, masterchain_handle_->id(),
masterchain_handle_->id(), write_data, std::move(P));
current_status_ = PSTRING() << "serializing masterchain state " << state->get_block_id().id.to_str();
current_status_ts_ = td::Timestamp::now();
}
void AsyncStateSerializer::stored_masterchain_state() {
current_status_ = "pending";
current_status_ts_ = {};
LOG(ERROR) << "finished serializing masterchain state " << masterchain_handle_->id().id.to_str();
running_ = false;
next_iteration();
@ -444,9 +458,14 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardStat
});
td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(),
masterchain_handle_->id(), write_data, std::move(P));
current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " "
<< state->get_block_id().id.to_str();
current_status_ts_ = td::Timestamp::now();
}
void AsyncStateSerializer::fail_handler(td::Status reason) {
current_status_ = PSTRING() << "pending, " << reason;
current_status_ts_ = {};
VLOG(VALIDATOR_NOTICE) << "failure: " << reason;
attempt_++;
delay_action(
@ -460,6 +479,8 @@ void AsyncStateSerializer::fail_handler_cont() {
}
void AsyncStateSerializer::success_handler() {
current_status_ = "pending";
current_status_ts_ = {};
running_ = false;
next_iteration();
}
@ -478,6 +499,29 @@ void AsyncStateSerializer::auto_disable_serializer(bool disabled) {
}
}
void AsyncStateSerializer::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
if (!inited_block_id_) {
wait_init_block_id_.push_back(
[SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::Unit> R) mutable {
TRY_STATUS_PROMISE(promise, R.move_as_status());
td::actor::send_closure(SelfId, &AsyncStateSerializer::prepare_stats, std::move(promise));
});
return;
}
std::vector<std::pair<std::string, std::string>> vec;
vec.emplace_back("stateserializermasterchainseqno", td::to_string(last_block_id_.seqno()));
td::StringBuilder sb;
sb << current_status_;
if (current_status_ts_) {
sb << " (started " << (int)(td::Timestamp::now() - current_status_ts_) << "s ago)";
}
if (!opts_->get_state_serializer_enabled() || auto_disabled_) {
sb << " (disabled)";
}
vec.emplace_back("stateserializerstatus", sb.as_cslice().str());
promise.set_result(std::move(vec));
}
bool AsyncStateSerializer::need_serialize(BlockHandle handle) {
if (handle->id().id.seqno == 0 || !handle->is_key_block()) {
return false;

8
validator/state-serializer.hpp
View file

@ -36,6 +36,9 @@ class AsyncStateSerializer : public td::actor::Actor {
UnixTime last_key_block_ts_ = 0;
bool saved_to_db_ = true;
bool inited_block_id_ = false;
std::vector<td::Promise<td::Unit>> wait_init_block_id_;
td::Ref<ValidatorManagerOptions> opts_;
bool auto_disabled_ = false;
td::CancellationTokenSource cancellation_token_source_;
@ -95,6 +98,8 @@ class AsyncStateSerializer : public td::actor::Actor {
promise.set_result(last_block_id_.id.seqno);
}
void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise);
void update_last_known_key_block_ts(UnixTime ts) {
last_known_key_block_ts_ = std::max(last_known_key_block_ts_, ts);
}
@ -111,6 +116,9 @@ class AsyncStateSerializer : public td::actor::Actor {
void update_options(td::Ref<ValidatorManagerOptions> opts);
void auto_disable_serializer(bool disabled);
std::string current_status_ = "pending";
td::Timestamp current_status_ts_ = td::Timestamp::never();
};
} // namespace validator

105
validator/stats-provider.h Normal file
View file

@ -0,0 +1,105 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "validator.h"
#include "common/AtomicRef.h"
#include <atomic>
namespace ton {
namespace validator {
class StatsProvider {
public:
StatsProvider() = default;
StatsProvider(td::actor::ActorId<ValidatorManagerInterface> manager, std::string prefix,
std::function<void(td::Promise<std::vector<std::pair<std::string, std::string>>>)> callback)
: inited_(true), manager_(std::move(manager)) {
static std::atomic<td::uint64> cur_idx{0};
idx_ = cur_idx.fetch_add(1);
td::actor::send_closure(manager_, &ValidatorManagerInterface::register_stats_provider, idx_, std::move(prefix),
std::move(callback));
}
StatsProvider(const StatsProvider&) = delete;
StatsProvider(StatsProvider&& other) noexcept
: inited_(other.inited_), idx_(other.idx_), manager_(std::move(other.manager_)) {
other.inited_ = false;
}
~StatsProvider() {
if (inited_) {
td::actor::send_closure(manager_, &ValidatorManagerInterface::unregister_stats_provider, idx_);
}
}
StatsProvider& operator=(const StatsProvider&) = delete;
StatsProvider& operator=(StatsProvider&& other) noexcept {
if (this != &other) {
inited_ = other.inited_;
idx_ = other.idx_;
manager_ = std::move(other.manager_);
other.inited_ = false;
}
return *this;
}
bool inited() const {
return inited_;
}
private:
bool inited_ = false;
td::uint64 idx_ = 0;
td::actor::ActorId<ValidatorManagerInterface> manager_;
};
class ProcessStatus {
public:
ProcessStatus() = default;
ProcessStatus(td::actor::ActorId<ValidatorManagerInterface> manager, std::string name)
: stats_provider_(std::move(manager), std::move(name), [value = value_](auto promise) {
auto status = value->load();
if (status.is_null()) {
promise.set_error(td::Status::Error("empty"));
return;
}
std::vector<std::pair<std::string, std::string>> vec;
vec.emplace_back("", *status);
promise.set_value(std::move(vec));
}) {
}
ProcessStatus(const ProcessStatus&) = delete;
ProcessStatus(ProcessStatus&& other) noexcept = default;
ProcessStatus& operator=(const ProcessStatus&) = delete;
ProcessStatus& operator=(ProcessStatus&& other) noexcept = default;
void set_status(std::string s) {
if (!value_) {
return;
}
value_->store(td::Ref<td::Cnt<std::string>>(true, std::move(s)));
}
private:
std::shared_ptr<td::AtomicRef<td::Cnt<std::string>>> value_ = std::make_shared<td::AtomicRef<td::Cnt<std::string>>>();
StatsProvider stats_provider_;
};
} // namespace validator
} // namespace ton

1
validator/validator-group.cpp
View file

@ -360,6 +360,7 @@ void ValidatorGroup::create_session() {
}
CHECK(found);
config_.catchain_opts.broadcast_speed_multiplier = opts_->get_catchain_broadcast_speed_multiplier();
if (!config_.new_catchain_ids) {
session_ = validatorsession::ValidatorSession::create(session_id_, config_, local_id_, std::move(vec),
make_validator_session_callback(), keyring_, adnl_, rldp_,

7
validator/validator-options.hpp
View file

@ -154,6 +154,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
bool get_fast_state_serializer_enabled() const override {
return fast_state_serializer_enabled_;
}
double get_catchain_broadcast_speed_multiplier() const override {
return catchain_broadcast_speed_multipliers_;
}
td::Ref<CollatorsList> get_collators_list() const override {
return collators_list_;
}
@ -255,6 +258,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
void set_fast_state_serializer_enabled(bool value) override {
fast_state_serializer_enabled_ = value;
}
void set_catchain_broadcast_speed_multiplier(double value) override {
catchain_broadcast_speed_multipliers_ = value;
}
void set_collators_list(td::Ref<CollatorsList> list) override {
collators_list_ = std::move(list);
}
@ -321,6 +327,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
bool state_serializer_enabled_ = true;
td::Ref<CollatorOptions> collator_options_{true};
bool fast_state_serializer_enabled_ = false;
double catchain_broadcast_speed_multipliers_;
td::Ref<CollatorsList> collators_list_{true, CollatorsList::default_list()};
std::set<adnl::AdnlNodeIdShort> collator_node_whitelist_;
bool collator_node_whitelist_enabled_ = false;

10
validator/validator.h
View file

@ -20,6 +20,7 @@
#include <vector>
#include <deque>
#include <functional>
#include "td/actor/actor.h"
@ -135,6 +136,7 @@ struct ValidatorManagerOptions : public td::CntObject {
virtual bool get_state_serializer_enabled() const = 0;
virtual td::Ref<CollatorOptions> get_collator_options() const = 0;
virtual bool get_fast_state_serializer_enabled() const = 0;
virtual double get_catchain_broadcast_speed_multiplier() const = 0;
virtual td::Ref<CollatorsList> get_collators_list() const = 0;
virtual bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const = 0;
@ -169,6 +171,7 @@ struct ValidatorManagerOptions : public td::CntObject {
virtual void set_state_serializer_enabled(bool value) = 0;
virtual void set_collator_options(td::Ref<CollatorOptions> value) = 0;
virtual void set_fast_state_serializer_enabled(bool value) = 0;
virtual void set_catchain_broadcast_speed_multiplier(double value) = 0;
virtual void set_collators_list(td::Ref<CollatorsList> list) = 0;
virtual void set_collator_node_whitelisted_validator(adnl::AdnlNodeIdShort id, bool add) = 0;
virtual void set_collator_node_whitelist_enabled(bool enabled) = 0;
@ -325,6 +328,13 @@ class ValidatorManagerInterface : public td::actor::Actor {
virtual void update_options(td::Ref<ValidatorManagerOptions> opts) = 0;
virtual void register_stats_provider(
td::uint64 idx, std::string prefix,
std::function<void(td::Promise<std::vector<std::pair<std::string, std::string>>>)> callback) {
}
virtual void unregister_stats_provider(td::uint64 idx) {
}
virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0;
virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0;