external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-02-15 04:32:21 +00:00
Code Issues Releases Wiki Activity

Rework session stats

Browse source
This commit is contained in:
SpyCheese 2024-12-26 14:03:12 +03:00
parent 392cf64758
commit 71342bdcd4
24 changed files with 436 additions and 334 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
tl/generate/scheme/ton_api.tl
View file

@ -895,48 +895,66 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se
---types--- ---types---
validatorSession.collationStats actual_bytes:int actual_collated_data_bytes:int validatorStats.stats.producer flags:#
bytes:int gas:int lt_delta:int collated_data_bytes:int validator_id:int256 block_status:int
cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int candidate_id:flags.0?int256 block_id:flags.0?tonNode.blockIdExt collated_data_hash:flags.0?int256
limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int is_accepted:flags.0?Bool is_ours:flags.0?Bool
work_time:double cpu_work_time:double got_block_at:flags.0?double got_submit_at:flags.0?double gen_utime:flags.0?int comment:flags.0?string
serialized_size:int serialized_size_no_collated_data:int = validadorSession.CollationStats; collation_time:flags.1?double collated_at:flags.1?double collation_cached:flags.1?Bool self_collated:flags.1?Bool collator_node_id:flags.2?int256
validation_time:flags.3?double validated_at:flags.3?double validation_cached:flags.3?Bool
approved_weight:flags.0?long approved_33pct_at:flags.0?double approved_66pct_at:flags.0?double approvers:flags.0?string
signed_weight:flags.0?long signed_33pct_at:flags.0?double signed_66pct_at:flags.0?double signers:flags.0?string
serialize_time:flags.4?double deserialize_time:flags.4?double serialized_size:flags.4?int = validatorStats.stats.Producer;
validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256 validatorStats.stats.round started_at:double producers:(vector validatorStats.stats.producer) = validatorStats.stats.Round;
comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double
collation_time:double collated_at:double collation_cached:Bool
collation_work_time:double collation_cpu_work_time:double
collation_stats:validatorSession.collationStats
validation_time:double validated_at:double validation_cached:Bool
validation_work_time:double validation_cpu_work_time:double
gen_utime:double
approved_weight:long approved_33pct_at:double approved_66pct_at:double approvers:string
signed_weight:long signed_33pct_at:double signed_66pct_at:double signers:string
serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer;
validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; validatorStats.stats flags:#
session_id:int256 self:int256 block_id:tonNode.blockIdExt cc_seqno:int success:Bool timestamp:double
creator:flags.0?int256
total_validators:int total_weight:long
signatures:flags.0?int signatures_weight:flags.0?long
approve_signatures:flags.0?int approve_signatures_weight:flags.0?long
first_round:int rounds:(vector validatorStats.stats.round) = validatorStats.Stats;
validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self:int256 session_id:int256 cc_seqno:int validatorStats.blockLimitsStatus
creator:int256 total_validators:int total_weight:long bytes:int gas:int lt_delta:int collated_data_bytes:int
signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int
first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; limits_log:string = validatorStats.BlockLimitsStatus;
validatorStats.extMsgsStats
ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats;
validatorSession.statsCollatedBlock timestamp:double id:tonNode.blockIdExt stats:validatorSession.collationStats = validatorSession.StatsCollatedBlock; validatorStats.collatedBlock flags:#
block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double
bytes:int collated_data_bytes:int attempt:int
collator_node_id:flags.0?int256 validator_id:flags.1?int256
total_time:double work_time:double cpu_work_time:double time_stats:string
block_limits:validatorStats.blockLimitsStatus
ext_msgs_stats:validatorStats.extMsgsStats = validatorSession.stats.CollatedBlock;
validatorStats.validatedBlock
block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double
valid:Bool comment:string
bytes:int collated_data_bytes:int
total_time:double work_time:double cpu_work_time:double = validatorStats.ValidatedBlock;
validatorStats.newValidatorGroup.node id:int256 weight:long = validatorStats.newValidatorGroup.Node;
validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int
last_key_block_seqno:int started_at:double
self_idx:int nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup;
validatorStats.endValidatorGroup.node id:int256 catchain_blocks:int = validatorStats.endValidatorGroup.Node;
validatorStats.endValidatorGroup session_id:int256 timestamp:double
nodes:(vector validatorStats.endValidatorGroup.node) = validatorStats.EndValidatorGroup;
validatorStats.collatorNodeResponse collator_node_id:int256 validator_id:int256 timestamp:double
block_id:tonNode.blockIdExt original_block_id:tonNode.blockIdExt collated_data_hash:int256
= validatorStats.CollatorNodeResponse;
collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate;
collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate;
collatorNode.pong flags:# = collatorNode.Pong; collatorNode.pong flags:# = collatorNode.Pong;
collatorNode.error code:int message:string = collatorNode.Error; collatorNode.error code:int message:string = collatorNode.Error;
validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node;
validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int
last_key_block_seqno:int timestamp:double
self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats;
validatorSession.endValidatorGroupStats.node id:int256 catchain_blocks:int = validatorSession.endValidatorGroupStats.Node;
validatorSession.endValidatorGroupStats session_id:int256 timestamp:double
nodes:(vector validatorSession.endValidatorGroupStats.node) = validatorSession.EndValidatorGroupStats;
---functions--- ---functions---
collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt)
creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate;

BIN
tl/generate/scheme/ton_api.tlo
View file

Binary file not shown.

11
ton/ton-types.h
View file

@ -501,6 +501,17 @@ struct BlockCandidate {
} }
}; };
struct GeneratedCandidate {
BlockCandidate candidate;
bool is_cached = false;
bool self_collated = false;
td::Bits256 collator_node_id = td::Bits256::zero();
GeneratedCandidate clone() const {
return {candidate.clone(), is_cached, self_collated, collator_node_id};
}
};
struct BlockCandidatePriority { struct BlockCandidatePriority {
td::uint32 round{}; td::uint32 round{};
td::uint32 first_block_round{}; td::uint32 first_block_round{};

118
validator-session/validator-session-types.h
View file

@ -23,6 +23,8 @@
#include "adnl/adnl-node-id.hpp" #include "adnl/adnl-node-id.hpp"
#include "ton/ton-types.h" #include "ton/ton-types.h"
#include <ton/ton-tl.hpp>
namespace ton { namespace ton {
namespace validatorsession { namespace validatorsession {
@ -73,24 +75,25 @@ struct ValidatorSessionStats {
enum { status_none = 0, status_received = 1, status_rejected = 2, status_approved = 3 }; enum { status_none = 0, status_received = 1, status_rejected = 2, status_approved = 3 };
struct Producer { struct Producer {
PublicKeyHash id = PublicKeyHash::zero();
ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero();
int block_status = status_none; int block_status = status_none;
double block_timestamp = -1.0; PublicKeyHash validator_id = PublicKeyHash::zero();
td::Bits256 root_hash = td::Bits256::zero(); ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero();
td::Bits256 file_hash = td::Bits256::zero(); BlockIdExt block_id{workchainIdNotYet, 0, 0, td::Bits256::zero(), td::Bits256::zero()};
std::string comment; td::Bits256 collated_data_hash = td::Bits256::zero();
bool is_accepted = false; bool is_accepted = false;
bool is_ours = false; bool is_ours = false;
double got_block_at = -1.0;
double got_submit_at = -1.0; double got_submit_at = -1.0;
double collation_time = -1.0; td::int32 gen_utime = -1;
double validation_time = -1.0; std::string comment;
double collated_at = -1.0;
double validated_at = -1.0; double collation_time = -1.0, collated_at = -1.0;
bool collation_cached = false; bool collation_cached = false;
bool self_collated = false;
td::Bits256 collator_node_id = td::Bits256::zero();
double validation_time = -1.0, validated_at = -1.0;
bool validation_cached = false; bool validation_cached = false;
double gen_utime = -1.0;
std::vector<bool> approvers, signers; std::vector<bool> approvers, signers;
ValidatorWeight approved_weight = 0; ValidatorWeight approved_weight = 0;
@ -129,20 +132,52 @@ struct ValidatorSessionStats {
} }
} }
} }
tl_object_ptr<ton_api::validatorStats_stats_producer> tl() const {
std::string approvers_str(approvers.size(), '0');
for (size_t i = 0; i < approvers.size(); ++i) {
approvers_str[i] = '0' + approvers[i];
}
std::string signers_str(signers.size(), '0');
for (size_t i = 0; i < signers.size(); ++i) {
signers_str[i] = '0' + signers[i];
}
int flags =
(block_status != status_none ? ton_api::validatorStats_stats_producer::Flags::BLOCK_ID_MASK : 0) |
(collated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::COLLATED_AT_MASK : 0) |
(!collator_node_id.is_zero() ? ton_api::validatorStats_stats_producer::Flags::COLLATOR_NODE_ID_MASK : 0) |
(validated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::VALIDATED_AT_MASK : 0) |
(serialize_time >= 0.0 || deserialize_time >= 0.0 || serialized_size >= 0
? ton_api::validatorStats_stats_producer::Flags::SERIALIZE_TIME_MASK
: 0);
return create_tl_object<ton_api::validatorStats_stats_producer>(
flags, validator_id.bits256_value(), block_status, candidate_id, create_tl_block_id(block_id),
collated_data_hash, is_accepted, is_ours, got_block_at, got_submit_at, gen_utime, comment, collation_time,
collated_at, collation_cached, self_collated, collator_node_id, validation_time, validated_at,
validation_cached, approved_weight, approved_33pct_at, approved_66pct_at, std::move(approvers_str),
signed_weight, signed_33pct_at, signed_66pct_at, std::move(signers_str), serialize_time, deserialize_time,
serialized_size);
}
}; };
struct Round { struct Round {
double timestamp = -1.0; double started_at = -1.0;
std::vector<Producer> producers; std::vector<Producer> producers;
tl_object_ptr<ton_api::validatorStats_stats_round> tl() const {
std::vector<tl_object_ptr<ton_api::validatorStats_stats_producer>> producers_tl;
for (const auto &producer : producers) {
producers_tl.push_back(producer.tl());
}
return create_tl_object<ton_api::validatorStats_stats_round>(started_at, std::move(producers_tl));
}
}; };
td::uint32 first_round;
std::vector<Round> rounds;
bool success = false;
ValidatorSessionId session_id = ValidatorSessionId::zero(); ValidatorSessionId session_id = ValidatorSessionId::zero();
CatchainSeqno cc_seqno = 0;
double timestamp = -1.0;
PublicKeyHash self = PublicKeyHash::zero(); PublicKeyHash self = PublicKeyHash::zero();
BlockIdExt block_id{workchainIdNotYet, 0, 0, td::Bits256::zero(), td::Bits256::zero()};
CatchainSeqno cc_seqno = 0;
bool success = false;
double timestamp = -1.0;
PublicKeyHash creator = PublicKeyHash::zero(); PublicKeyHash creator = PublicKeyHash::zero();
td::uint32 total_validators = 0; td::uint32 total_validators = 0;
ValidatorWeight total_weight = 0; ValidatorWeight total_weight = 0;
@ -150,6 +185,29 @@ struct ValidatorSessionStats {
ValidatorWeight signatures_weight = 0; ValidatorWeight signatures_weight = 0;
td::uint32 approve_signatures = 0; td::uint32 approve_signatures = 0;
ValidatorWeight approve_signatures_weight = 0; ValidatorWeight approve_signatures_weight = 0;
td::uint32 first_round = 0;
std::vector<Round> rounds;
void fix_block_ids() {
for (auto &round : rounds) {
for (auto &producer : round.producers) {
producer.block_id.id = block_id.id;
}
}
}
tl_object_ptr<ton_api::validatorStats_stats> tl() const {
std::vector<tl_object_ptr<ton_api::validatorStats_stats_round>> rounds_tl;
for (const auto &round : rounds) {
rounds_tl.push_back(round.tl());
}
int flags = success ? ton_api::validatorStats_stats::Flags::CREATOR_MASK : 0;
return create_tl_object<ton_api::validatorStats_stats>(
flags, session_id, self.bits256_value(), create_tl_block_id(block_id), cc_seqno, success, timestamp,
creator.bits256_value(), total_validators, total_weight, signatures, signatures_weight, approve_signatures,
approve_signatures_weight, first_round, std::move(rounds_tl));
}
}; };
struct NewValidatorGroupStats { struct NewValidatorGroupStats {
@ -162,9 +220,20 @@ struct NewValidatorGroupStats {
ShardIdFull shard{masterchainId}; ShardIdFull shard{masterchainId};
CatchainSeqno cc_seqno = 0; CatchainSeqno cc_seqno = 0;
BlockSeqno last_key_block_seqno = 0; BlockSeqno last_key_block_seqno = 0;
double timestamp = -1.0; double started_at = -1.0;
td::uint32 self_idx = 0; td::uint32 self_idx = 0;
std::vector<Node> nodes; std::vector<Node> nodes;
tl_object_ptr<ton_api::validatorStats_newValidatorGroup> tl() const {
std::vector<tl_object_ptr<ton_api::validatorStats_newValidatorGroup_node>> nodes_arr;
for (const auto &node : nodes) {
nodes_arr.push_back(
create_tl_object<ton_api::validatorStats_newValidatorGroup_node>(node.id.bits256_value(), node.weight));
}
return create_tl_object<ton_api::validatorStats_newValidatorGroup>(session_id, create_tl_shard_id(shard), cc_seqno,
last_key_block_seqno, started_at, self_idx,
std::move(nodes_arr));
}
}; };
struct EndValidatorGroupStats { struct EndValidatorGroupStats {
@ -176,6 +245,15 @@ struct EndValidatorGroupStats {
ValidatorSessionId session_id = ValidatorSessionId::zero(); ValidatorSessionId session_id = ValidatorSessionId::zero();
double timestamp = -1.0; double timestamp = -1.0;
std::vector<Node> nodes; std::vector<Node> nodes;
tl_object_ptr<ton_api::validatorStats_endValidatorGroup> tl() const {
std::vector<tl_object_ptr<ton_api::validatorStats_endValidatorGroup_node>> nodes_arr;
for (const auto &node : nodes) {
nodes_arr.push_back(create_tl_object<ton_api::validatorStats_endValidatorGroup_node>(node.id.bits256_value(),
node.catchain_blocks));
}
return create_tl_object<ton_api::validatorStats_endValidatorGroup>(session_id, timestamp, std::move(nodes_arr));
}
}; };
struct BlockSourceInfo { struct BlockSourceInfo {

70
validator-session/validator-session.cpp
View file

@ -266,13 +266,14 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice
if (stat->block_status == ValidatorSessionStats::status_none) { if (stat->block_status == ValidatorSessionStats::status_none) {
stat->block_status = ValidatorSessionStats::status_received; stat->block_status = ValidatorSessionStats::status_received;
} }
if (stat->block_timestamp <= 0.0) { if (stat->got_block_at <= 0.0) {
stat->block_timestamp = td::Clocks::system(); stat->got_block_at = td::Clocks::system();
} }
stat->deserialize_time = deserialize_time; stat->deserialize_time = deserialize_time;
stat->serialized_size = data.size(); stat->serialized_size = data.size();
stat->root_hash = candidate->root_hash_; stat->block_id.root_hash = candidate->root_hash_;
stat->file_hash = file_hash; stat->block_id.file_hash = file_hash;
stat->collated_data_hash = collated_data_file_hash;
} }
if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK ||
@ -451,35 +452,36 @@ void ValidatorSessionImpl::candidate_approved_signed(td::uint32 round, Validator
} }
} }
void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCandidateId root_hash, void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate c, double collation_time) {
td::BufferSlice data, td::BufferSlice collated_data, double collation_time, if (c.candidate.data.size() > description().opts().max_block_size ||
bool collation_cached) { c.candidate.collated_data.size() > description().opts().max_collated_data_size) {
if (data.size() > description().opts().max_block_size || LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << c.candidate.data.size() << " "
collated_data.size() > description().opts().max_collated_data_size) { << c.candidate.collated_data.size();
LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << data.size() << " "
<< collated_data.size();
return; return;
} }
auto file_hash = sha256_bits256(data.as_slice()); auto file_hash = sha256_bits256(c.candidate.data.as_slice());
auto collated_data_file_hash = sha256_bits256(collated_data.as_slice()); auto collated_data_file_hash = sha256_bits256(c.candidate.collated_data.as_slice());
auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash); auto block_id = description().candidate_id(local_idx(), c.candidate.id.root_hash, file_hash, collated_data_file_hash);
auto stat = stats_get_candidate_stat(round, local_id(), block_id); auto stat = stats_get_candidate_stat(round, local_id(), block_id);
if (stat) { if (stat) {
stat->block_status = ValidatorSessionStats::status_received; stat->block_status = ValidatorSessionStats::status_received;
stat->collation_time = collation_time; stat->collation_time = collation_time;
stat->collated_at = td::Clocks::system(); stat->collated_at = td::Clocks::system();
stat->block_timestamp = td::Clocks::system(); stat->got_block_at = td::Clocks::system();
stat->collation_cached = collation_cached; stat->collation_cached = c.is_cached;
stat->root_hash = root_hash; stat->self_collated = c.self_collated;
stat->file_hash = file_hash; stat->collator_node_id = c.collator_node_id;
stat->block_id = c.candidate.id;
stat->collated_data_hash = collated_data_file_hash;
} }
if (round != cur_round_) { if (round != cur_round_) {
return; return;
} }
td::Timer serialize_timer; td::Timer serialize_timer;
auto b = create_tl_object<ton_api::validatorSession_candidate>(local_id().tl(), round, root_hash, std::move(data), auto b = create_tl_object<ton_api::validatorSession_candidate>(local_id().tl(), round, c.candidate.id.root_hash,
std::move(collated_data)); std::move(c.candidate.data),
std::move(c.candidate.collated_data));
auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok(); auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok();
if (stat) { if (stat) {
stat->serialize_time = serialize_timer.elapsed(); stat->serialize_time = serialize_timer.elapsed();
@ -546,9 +548,8 @@ void ValidatorSessionImpl::check_generate_slot() {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), timer = std::move(timer), auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), timer = std::move(timer),
round = cur_round_](td::Result<GeneratedCandidate> R) { round = cur_round_](td::Result<GeneratedCandidate> R) {
if (R.is_ok()) { if (R.is_ok()) {
auto c = std::move(R.ok_ref().candidate); td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, R.move_as_ok(),
td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, c.id.root_hash, timer.elapsed());
c.data.clone(), c.collated_data.clone(), timer.elapsed(), R.ok().is_cached);
} else { } else {
LOG(WARNING) << print_id << ": failed to generate block candidate: " << R.move_as_error(); LOG(WARNING) << print_id << ": failed to generate block candidate: " << R.move_as_error();
} }
@ -606,11 +607,12 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) {
if (stat->block_status == ValidatorSessionStats::status_none) { if (stat->block_status == ValidatorSessionStats::status_none) {
stat->block_status = ValidatorSessionStats::status_received; stat->block_status = ValidatorSessionStats::status_received;
} }
if (stat->block_timestamp <= 0.0) { if (stat->got_block_at <= 0.0) {
stat->block_timestamp = td::Clocks::system(); stat->got_block_at = td::Clocks::system();
} }
stat->root_hash = B->root_hash_; stat->block_id.root_hash = B->root_hash_;
stat->file_hash = td::sha256_bits256(B->data_); stat->block_id.file_hash = td::sha256_bits256(B->data_);
stat->collated_data_hash = td::sha256_bits256(B->collated_data_);
} }
auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(), auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(),
@ -933,7 +935,7 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) {
while (round_idx >= cur_stats_.rounds.size()) { while (round_idx >= cur_stats_.rounds.size()) {
stats_add_round(); stats_add_round();
} }
cur_stats_.rounds[round_idx].timestamp = td::Clocks::system(); cur_stats_.rounds[round_idx].started_at = td::Clocks::system();
} }
auto it2 = blocks_.begin(); auto it2 = blocks_.begin();
while (it2 != blocks_.end()) { while (it2 != blocks_.end()) {
@ -1034,9 +1036,7 @@ void ValidatorSessionImpl::get_end_stats(td::Promise<EndValidatorGroupStats> pro
promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); promise.set_error(td::Status::Error(ErrorCode::notready, "not started"));
return; return;
} }
EndValidatorGroupStats stats; EndValidatorGroupStats stats{.session_id = unique_hash_, .timestamp = td::Clocks::system()};
stats.session_id = unique_hash_;
stats.timestamp = td::Clocks::system();
stats.nodes.resize(description().get_total_nodes()); stats.nodes.resize(description().get_total_nodes());
for (size_t i = 0; i < stats.nodes.size(); ++i) { for (size_t i = 0; i < stats.nodes.size(); ++i) {
stats.nodes[i].id = description().get_source_id(i); stats.nodes[i].id = description().get_source_id(i);
@ -1145,7 +1145,7 @@ void ValidatorSessionImpl::stats_init() {
if (cur_stats_.rounds.empty()) { if (cur_stats_.rounds.empty()) {
stats_add_round(); stats_add_round();
} }
cur_stats_.rounds[0].timestamp = td::Clocks::system(); cur_stats_.rounds[0].started_at = td::Clocks::system();
stats_inited_ = true; stats_inited_ = true;
} }
@ -1158,13 +1158,13 @@ void ValidatorSessionImpl::stats_add_round() {
td::int32 priority = description().get_node_priority(i, round); td::int32 priority = description().get_node_priority(i, round);
if (priority >= 0) { if (priority >= 0) {
CHECK((size_t)priority < stat.producers.size()); CHECK((size_t)priority < stat.producers.size());
stat.producers[priority].id = description().get_source_id(i); stat.producers[priority].validator_id = description().get_source_id(i);
stat.producers[priority].is_ours = (local_idx() == i); stat.producers[priority].is_ours = (local_idx() == i);
stat.producers[priority].approvers.resize(description().get_total_nodes(), false); stat.producers[priority].approvers.resize(description().get_total_nodes(), false);
stat.producers[priority].signers.resize(description().get_total_nodes(), false); stat.producers[priority].signers.resize(description().get_total_nodes(), false);
} }
} }
while (!stat.producers.empty() && stat.producers.back().id.is_zero()) { while (!stat.producers.empty() && stat.producers.back().validator_id.is_zero()) {
stat.producers.pop_back(); stat.producers.pop_back();
} }
} }
@ -1179,7 +1179,7 @@ ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat(
} }
auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round]; auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round];
auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(), auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(),
[&](const ValidatorSessionStats::Producer &p) { return p.id == src; }); [&](const ValidatorSessionStats::Producer &p) { return p.validator_id == src; });
if (it == stats_round.producers.end()) { if (it == stats_round.producers.end()) {
return nullptr; return nullptr;
} }

5
validator-session/validator-session.h
View file

@ -78,11 +78,6 @@ class ValidatorSession : public td::actor::Actor {
bool is_cached_ = false; bool is_cached_ = false;
}; };
struct GeneratedCandidate {
BlockCandidate candidate;
bool is_cached = false;
};
class Callback { class Callback {
public: public:
virtual void on_candidate(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, td::BufferSlice data, virtual void on_candidate(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, td::BufferSlice data,

3
validator-session/validator-session.hpp
View file

@ -217,8 +217,7 @@ class ValidatorSessionImpl : public ValidatorSession {
void candidate_approved_signed(td::uint32 round, ValidatorSessionCandidateId hash, td::uint32 ok_from, void candidate_approved_signed(td::uint32 round, ValidatorSessionCandidateId hash, td::uint32 ok_from,
td::BufferSlice signature); td::BufferSlice signature);
void generated_block(td::uint32 round, ValidatorSessionRootHash root_hash, td::BufferSlice data, void generated_block(td::uint32 round, GeneratedCandidate c, double collation_time);
td::BufferSlice collated, double collation_time, bool collation_cached);
void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature); void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature);
void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) { void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) {

64
validator/collation-manager.cpp
View file

@ -33,13 +33,16 @@ void CollationManager::start_up() {
void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id,
std::vector<BlockIdExt> prev, Ed25519_PublicKey creator, std::vector<BlockIdExt> prev, Ed25519_PublicKey creator,
BlockCandidatePriority priority, BlockCandidatePriority priority, td::Ref<ValidatorSet> validator_set,
td::Ref<ValidatorSet> validator_set, td::uint64 max_answer_size, td::uint64 max_answer_size, td::CancellationToken cancellation_token,
td::CancellationToken cancellation_token, td::Promise<BlockCandidate> promise) { td::Promise<GeneratedCandidate> promise) {
if (shard.is_masterchain()) { if (shard.is_masterchain()) {
run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), run_collate_query(
opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set),
std::move(cancellation_token), 0); opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) {
return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true};
}),
adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0);
return; return;
} }
collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set),
@ -48,10 +51,9 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha
void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id,
std::vector<BlockIdExt> prev, Ed25519_PublicKey creator, std::vector<BlockIdExt> prev, Ed25519_PublicKey creator,
BlockCandidatePriority priority, BlockCandidatePriority priority, td::Ref<ValidatorSet> validator_set,
td::Ref<ValidatorSet> validator_set, td::uint64 max_answer_size, td::uint64 max_answer_size, td::CancellationToken cancellation_token,
td::CancellationToken cancellation_token, td::Promise<GeneratedCandidate> promise, td::Timestamp timeout) {
td::Promise<BlockCandidate> promise, td::Timestamp timeout) {
TRY_STATUS_PROMISE(promise, cancellation_token.check()); TRY_STATUS_PROMISE(promise, cancellation_token.check());
ShardInfo* s = select_shard_info(shard); ShardInfo* s = select_shard_info(shard);
if (s == nullptr) { if (s == nullptr) {
@ -104,9 +106,12 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas
} }
if (selected_collator.is_zero() && s->self_collate) { if (selected_collator.is_zero() && s->self_collate) {
run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), run_collate_query(
opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set),
std::move(cancellation_token), 0); opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) {
return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true};
}),
adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0);
return; return;
} }
@ -117,10 +122,13 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas
next_block_id.seqno = std::max(next_block_id.seqno, p.seqno() + 1); next_block_id.seqno = std::max(next_block_id.seqno, p.seqno() + 1);
} }
promise = [=, SelfId = actor_id(this), promise = std::move(promise), td::Promise<BlockCandidate> P = [=, SelfId = actor_id(this), promise = std::move(promise),
retry_at = td::Timestamp::in(0.5)](td::Result<BlockCandidate> R) mutable { retry_at = td::Timestamp::in(0.5)](td::Result<BlockCandidate> R) mutable {
if (R.is_ok()) { if (R.is_ok()) {
promise.set_value(R.move_as_ok()); promise.set_value(GeneratedCandidate{.candidate = R.move_as_ok(),
.is_cached = false,
.self_collated = false,
.collator_node_id = selected_collator.bits256_value()});
return; return;
} }
if (!selected_collator.is_zero()) { if (!selected_collator.is_zero()) {
@ -142,7 +150,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas
}; };
if (selected_collator.is_zero()) { if (selected_collator.is_zero()) {
promise.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node")); P.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node"));
return; return;
} }
@ -152,32 +160,32 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas
LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "(" LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "("
<< selected_collator << ")"; << selected_collator << ")";
td::Promise<td::BufferSlice> P = [=, SelfId = actor_id(this), promise = std::move(promise), td::Promise<td::BufferSlice> P2 = [=, SelfId = actor_id(this), P = std::move(P),
timer = td::Timer()](td::Result<td::BufferSlice> R) mutable { timer = td::Timer()](td::Result<td::BufferSlice> R) mutable {
TRY_RESULT_PROMISE_PREFIX(promise, data, std::move(R), "rldp query failed: "); TRY_RESULT_PROMISE_PREFIX(P, data, std::move(R), "rldp query failed: ");
auto r_error = fetch_tl_object<ton_api::collatorNode_error>(data, true); auto r_error = fetch_tl_object<ton_api::collatorNode_error>(data, true);
if (r_error.is_ok()) { if (r_error.is_ok()) {
auto error = r_error.move_as_ok(); auto error = r_error.move_as_ok();
promise.set_error(td::Status::Error(error->code_, error->message_)); P.set_error(td::Status::Error(error->code_, error->message_));
return; return;
} }
TRY_RESULT_PROMISE(promise, f, fetch_tl_object<ton_api::collatorNode_Candidate>(data, true)); TRY_RESULT_PROMISE(P, f, fetch_tl_object<ton_api::collatorNode_Candidate>(data, true));
TRY_RESULT_PROMISE(promise, candidate, TRY_RESULT_PROMISE(P, candidate,
CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast<int>(max_answer_size))); CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast<int>(max_answer_size)));
if (candidate.pubkey.as_bits256() != creator.as_bits256()) { if (candidate.pubkey.as_bits256() != creator.as_bits256()) {
promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); P.set_error(td::Status::Error("collate query: block candidate source mismatch"));
return; return;
} }
if (candidate.id.id != next_block_id) { if (candidate.id.id != next_block_id) {
promise.set_error(td::Status::Error("collate query: block id mismatch")); P.set_error(td::Status::Error("collate query: block id mismatch"));
return; return;
} }
LOG(INFO) << "got collated block " << next_block_id.to_str() << " from #" << selected_idx << " (" LOG(INFO) << "got collated block " << next_block_id.to_str() << " from #" << selected_idx << " ("
<< selected_collator << ") in " << timer.elapsed() << "s"; << selected_collator << ") in " << timer.elapsed() << "s";
promise.set_result(std::move(candidate)); P.set_result(std::move(candidate));
}; };
td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", std::move(P), td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery",
timeout, std::move(query), max_answer_size); std::move(P2), timeout, std::move(query), max_answer_size);
} }
void CollationManager::update_options(td::Ref<ValidatorManagerOptions> opts) { void CollationManager::update_options(td::Ref<ValidatorManagerOptions> opts) {

4
validator/collation-manager.hpp
View file

@ -37,7 +37,7 @@ class CollationManager : public td::actor::Actor {
void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev, void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev,
Ed25519_PublicKey creator, BlockCandidatePriority priority, Ed25519_PublicKey creator, BlockCandidatePriority priority,
td::Ref<ValidatorSet> validator_set, td::uint64 max_answer_size, td::Ref<ValidatorSet> validator_set, td::uint64 max_answer_size,
td::CancellationToken cancellation_token, td::Promise<BlockCandidate> promise); td::CancellationToken cancellation_token, td::Promise<GeneratedCandidate> promise);
void update_options(td::Ref<ValidatorManagerOptions> opts); void update_options(td::Ref<ValidatorManagerOptions> opts);
@ -55,7 +55,7 @@ class CollationManager : public td::actor::Actor {
void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev, void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev,
Ed25519_PublicKey creator, BlockCandidatePriority priority, Ed25519_PublicKey creator, BlockCandidatePriority priority,
td::Ref<ValidatorSet> validator_set, td::uint64 max_answer_size, td::Ref<ValidatorSet> validator_set, td::uint64 max_answer_size,
td::CancellationToken cancellation_token, td::Promise<BlockCandidate> promise, td::CancellationToken cancellation_token, td::Promise<GeneratedCandidate> promise,
td::Timestamp timeout); td::Timestamp timeout);
void update_collators_list(const CollatorsList& collators_list); void update_collators_list(const CollatorsList& collators_list);

16
validator/collator-node.cpp
View file

@ -391,12 +391,24 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data
promise.set_result(serialize_tl_object(serialize_candidate(R.move_as_ok(), true), true)); promise.set_result(serialize_tl_object(serialize_candidate(R.move_as_ok(), true), true));
} }
}; };
new_promise = [new_promise = std::move(new_promise), creator, new_promise = [new_promise = std::move(new_promise), creator, local_id = local_id_,
manager = manager_](td::Result<BlockCandidate> R) mutable { manager = manager_](td::Result<BlockCandidate> R) mutable {
TRY_RESULT_PROMISE(new_promise, block, std::move(R)); TRY_RESULT_PROMISE(new_promise, block, std::move(R));
CollatorNodeResponseStats stats;
stats.collator_node_id = local_id.bits256_value();
stats.validator_id = creator.as_bits256();
stats.original_block_id = block.id;
stats.collated_data_hash = block.collated_file_hash;
CatchainSeqno cc_seqno; CatchainSeqno cc_seqno;
td::uint32 val_set_hash; td::uint32 val_set_hash;
block = change_creator(std::move(block), creator, cc_seqno, val_set_hash); block = change_creator(std::move(block), creator, cc_seqno, val_set_hash);
stats.block_id = block.id;
stats.timestamp = td::Clocks::system();
td::actor::send_closure(manager, &ValidatorManager::log_collator_node_response_stats, std::move(stats));
td::Promise<td::Unit> P = td::Promise<td::Unit> P =
new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); }); new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); });
td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno, td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno,
@ -537,7 +549,7 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std
}; };
td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R));
}, },
cache_entry->cancellation_token_source.get_cancellation_token(), local_id_, cache_entry->cancellation_token_source.get_cancellation_token(),
CollateMode::skip_store_candidate | CollateMode::from_collator_node); CollateMode::skip_store_candidate | CollateMode::from_collator_node);
} }

3
validator/fabric.h
View file

@ -89,7 +89,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc
Ed25519_PublicKey creator, td::Ref<ValidatorSet> validator_set, Ed25519_PublicKey creator, td::Ref<ValidatorSet> validator_set,
td::Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager, td::Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager,
td::Timestamp timeout, td::Promise<BlockCandidate> promise, td::Timestamp timeout, td::Promise<BlockCandidate> promise,
td::CancellationToken cancellation_token, unsigned mode, int attempt_idx = 0); adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode,
int attempt_idx = 0);
void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector<BlockIdExt> prev, void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector<BlockIdExt> prev,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout, td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
td::Promise<BlockCandidate> promise); td::Promise<BlockCandidate> promise);

4
validator/impl/collator-impl.h
View file

@ -79,6 +79,7 @@ class Collator final : public td::actor::Actor {
td::Timestamp timeout; td::Timestamp timeout;
td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_;
td::Promise<BlockCandidate> main_promise; td::Promise<BlockCandidate> main_promise;
adnl::AdnlNodeIdShort collator_node_id_ = adnl::AdnlNodeIdShort::zero();
unsigned mode_ = 0; unsigned mode_ = 0;
int attempt_idx_; int attempt_idx_;
bool allow_repeat_collation_ = false; bool allow_repeat_collation_ = false;
@ -97,7 +98,8 @@ class Collator final : public td::actor::Actor {
Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev, Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev,
Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id, Ref<CollatorOptions> collator_opts, Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id, Ref<CollatorOptions> collator_opts,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout, td::Promise<BlockCandidate> promise, td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout, td::Promise<BlockCandidate> promise,
td::CancellationToken cancellation_token, unsigned mode, int attempt_idx); adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode,
int attempt_idx);
~Collator() override = default; ~Collator() override = default;
bool is_busy() const { bool is_busy() const {
return busy_; return busy_;

31
validator/impl/collator.cpp
View file

@ -77,6 +77,7 @@ static inline bool dbg(int c) {
* @param manager The ActorId of the ValidatorManager. * @param manager The ActorId of the ValidatorManager.
* @param timeout The timeout for the collator. * @param timeout The timeout for the collator.
* @param promise The promise to return the result. * @param promise The promise to return the result.
* @param collator_node_id ADNL id of the collator node that generates the block (zero if it's not a collator node)
* @param cancellation_token Token to cancel collation. * @param cancellation_token Token to cancel collation.
* @param mode +1 - skip storing candidate to disk, +2 - called from CollatorNode. * @param mode +1 - skip storing candidate to disk, +2 - called from CollatorNode.
* @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps. * @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps.
@ -84,8 +85,8 @@ static inline bool dbg(int c) {
Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id,
std::vector<BlockIdExt> prev, td::Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id, std::vector<BlockIdExt> prev, td::Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id,
Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager, Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager,
td::Timestamp timeout, td::Promise<BlockCandidate> promise, td::CancellationToken cancellation_token, td::Timestamp timeout, td::Promise<BlockCandidate> promise, adnl::AdnlNodeIdShort collator_node_id,
unsigned mode, int attempt_idx) td::CancellationToken cancellation_token, unsigned mode, int attempt_idx)
: shard_(shard) : shard_(shard)
, is_hardfork_(is_hardfork) , is_hardfork_(is_hardfork)
, min_mc_block_id{min_masterchain_block_id} , min_mc_block_id{min_masterchain_block_id}
@ -100,6 +101,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha
, soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0))
, medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5))
, main_promise(std::move(promise)) , main_promise(std::move(promise))
, collator_node_id_(collator_node_id)
, mode_(mode) , mode_(mode)
, attempt_idx_(attempt_idx) , attempt_idx_(attempt_idx)
, perf_timer_("collate", 0.1, , perf_timer_("collate", 0.1,
@ -374,8 +376,8 @@ bool Collator::fatal_error(td::Status error) {
!is_hardfork_ && !timeout.is_in_past()) { !is_hardfork_ && !timeout.is_in_past()) {
LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")"; LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")";
run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager, run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager,
td::Timestamp::in(10.0), std::move(main_promise), std::move(cancellation_token_), mode_, td::Timestamp::in(10.0), std::move(main_promise), collator_node_id_,
attempt_idx_ + 1); std::move(cancellation_token_), mode_, attempt_idx_ + 1);
} else { } else {
LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error; LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error;
LOG(INFO) << perf_log_; LOG(INFO) << perf_log_;
@ -5927,8 +5929,15 @@ bool Collator::create_block_candidate() {
double work_time = work_timer_.elapsed(); double work_time = work_timer_.elapsed();
double cpu_work_time = cpu_work_timer_.elapsed(); double cpu_work_time = cpu_work_timer_.elapsed();
LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s";
stats_.block_id = block_candidate->id;
stats_.collated_data_hash = block_candidate->collated_file_hash;
stats_.cc_seqno = validator_set_->get_catchain_seqno();
stats_.collated_at = td::Clocks::system();
stats_.actual_bytes = block_candidate->data.size(); stats_.actual_bytes = block_candidate->data.size();
stats_.actual_collated_data_bytes = block_candidate->collated_data.size(); stats_.actual_collated_data_bytes = block_candidate->collated_data.size();
stats_.attempt = attempt_idx_;
stats_.collator_node_id = collator_node_id_.bits256_value();
stats_.validator_id = created_by_.as_bits256();
stats_.estimated_bytes = block_limit_status_->estimate_block_size(); stats_.estimated_bytes = block_limit_status_->estimate_block_size();
stats_.gas = block_limit_status_->gas_used; stats_.gas = block_limit_status_->gas_used;
stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt;
@ -5938,20 +5947,12 @@ bool Collator::create_block_candidate() {
stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt);
stats_.cat_collated_data_bytes = stats_.cat_collated_data_bytes =
block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes);
stats_.total_time = perf_timer_.elapsed();
stats_.work_time = work_time; stats_.work_time = work_time;
stats_.cpu_work_time = cpu_work_time; stats_.cpu_work_time = cpu_work_time;
stats_.time_stats = (PSTRING() << perf_log_);
// TODO: remove this later (currently needed to collect stats) td::actor::send_closure(manager, &ValidatorManager::log_collate_query_stats, std::move(stats_));
if (mode_ & CollateMode::from_collator_node) {
size_t d;
stats_.serialized_size =
validatorsession::compress_candidate_data(block_candidate->data, block_candidate->collated_data, d).ok().size();
stats_.serialized_size_no_collated_data =
validatorsession::compress_candidate_data(block_candidate->data, td::Slice{}, d).ok().size();
}
td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id,
std::move(stats_));
return true; return true;
} }

13
validator/impl/fabric.cpp
View file

@ -216,7 +216,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc
Ed25519_PublicKey creator, td::Ref<ValidatorSet> validator_set, Ed25519_PublicKey creator, td::Ref<ValidatorSet> validator_set,
td::Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager, td::Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager,
td::Timestamp timeout, td::Promise<BlockCandidate> promise, td::Timestamp timeout, td::Promise<BlockCandidate> promise,
td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) { adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode,
int attempt_idx) {
BlockSeqno seqno = 0; BlockSeqno seqno = 0;
for (auto& p : prev) { for (auto& p : prev) {
if (p.seqno() > seqno) { if (p.seqno() > seqno) {
@ -227,7 +228,7 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc
<< (attempt_idx ? "_" + td::to_string(attempt_idx) : ""), << (attempt_idx ? "_" + td::to_string(attempt_idx) : ""),
shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set), shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set),
creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise), creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise),
std::move(cancellation_token), mode, attempt_idx) collator_node_id, std::move(cancellation_token), mode, attempt_idx)
.release(); .release();
} }
@ -240,10 +241,10 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b
seqno = p.seqno(); seqno = p.seqno();
} }
} }
td::actor::create_actor<Collator>(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, td::actor::create_actor<Collator>(
min_masterchain_block_id, std::move(prev), td::Ref<ValidatorSet>{}, PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id,
Ed25519_PublicKey{Bits256::zero()}, td::Ref<CollatorOptions>{true}, std::move(prev), td::Ref<ValidatorSet>{}, Ed25519_PublicKey{Bits256::zero()}, td::Ref<CollatorOptions>{true},
std::move(manager), timeout, std::move(promise), td::CancellationToken{}, 0, 0) std::move(manager), timeout, std::move(promise), adnl::AdnlNodeIdShort::zero(), td::CancellationToken{}, 0, 0)
.release(); .release();
} }

36
validator/impl/validate-query.cpp
View file

@ -116,13 +116,12 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) {
error = error_ctx() + error; error = error_ctx() + error;
LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error;
if (main_promise) { if (main_promise) {
record_stats(); record_stats(false, error);
errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str() errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str()
<< " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex()
<< " collated_data=" << block_candidate.collated_file_hash.to_hex()); << " collated_data=" << block_candidate.collated_file_hash.to_hex());
errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.data.clone());
errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone());
LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s";
main_promise.set_result(CandidateReject{std::move(error), std::move(reason)}); main_promise.set_result(CandidateReject{std::move(error), std::move(reason)});
} }
stop(); stop();
@ -155,13 +154,12 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason)
error = error_ctx() + error; error = error_ctx() + error;
LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error;
if (main_promise) { if (main_promise) {
record_stats(); record_stats(false, error);
errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str()
<< " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex()
<< " collated_data=" << block_candidate.collated_file_hash.to_hex()); << " collated_data=" << block_candidate.collated_file_hash.to_hex());
errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.data.clone());
errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone());
LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s";
main_promise.set_result(CandidateReject{std::move(error), std::move(reason)}); main_promise.set_result(CandidateReject{std::move(error), std::move(reason)});
} }
stop(); stop();
@ -179,7 +177,7 @@ bool ValidateQuery::fatal_error(td::Status error) {
error.ensure_error(); error.ensure_error();
LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string(); LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string();
if (main_promise) { if (main_promise) {
record_stats(); record_stats(false, error.message().str());
auto c = error.code(); auto c = error.code();
if (c <= -667 && c >= -670) { if (c <= -667 && c >= -670) {
errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str() errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str()
@ -188,7 +186,6 @@ bool ValidateQuery::fatal_error(td::Status error) {
errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.data.clone());
errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone());
} }
LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s";
main_promise(std::move(error)); main_promise(std::move(error));
} }
stop(); stop();
@ -238,9 +235,8 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) {
*/ */
void ValidateQuery::finish_query() { void ValidateQuery::finish_query() {
if (main_promise) { if (main_promise) {
record_stats(); record_stats(true);
LOG(WARNING) << "validate query done"; LOG(WARNING) << "validate query done";
LOG(WARNING) << "validation took " << perf_timer_.elapsed() << " s";
main_promise.set_result(now_); main_promise.set_result(now_);
} }
stop(); stop();
@ -7061,13 +7057,25 @@ void ValidateQuery::written_candidate() {
/** /**
* Sends validation work time to manager. * Sends validation work time to manager.
*/ */
void ValidateQuery::record_stats() { void ValidateQuery::record_stats(bool valid, std::string error_message) {
double work_time = work_timer_.elapsed(); ValidationStats stats;
double cpu_work_time = cpu_work_timer_.elapsed(); stats.block_id = id_;
stats.collated_data_hash = block_candidate.collated_file_hash;
stats.validated_at = td::Clocks::system();
stats.valid = valid;
if (valid) {
stats.comment = (PSTRING() << "OK ts=" << now_);
} else {
stats.comment = std::move(error_message);
}
stats.actual_bytes = block_candidate.data.size();
stats.actual_collated_data_bytes = block_candidate.collated_data.size();
stats.total_time = perf_timer_.elapsed();
stats.work_time = work_timer_.elapsed();
stats.cpu_work_time = cpu_work_timer_.elapsed();
LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s";
LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; LOG(WARNING) << "Validate query work time = " << stats.work_time << "s, cpu time = " << stats.cpu_work_time << "s";
td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time, td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats));
cpu_work_time);
} }
} // namespace validator } // namespace validator

2
validator/impl/validate-query.hpp
View file

@ -407,7 +407,7 @@ class ValidateQuery : public td::actor::Actor {
td::Timer work_timer_{true}; td::Timer work_timer_{true};
td::ThreadCpuTimer cpu_work_timer_{true}; td::ThreadCpuTimer cpu_work_timer_{true};
void record_stats(); void record_stats(bool valid, std::string error_message = "");
}; };
} // namespace validator } // namespace validator

76
validator/interfaces/validator-manager.h
View file

@ -32,6 +32,8 @@
#include "auto/tl/lite_api.h" #include "auto/tl/lite_api.h"
#include "impl/out-msg-queue-proof.hpp" #include "impl/out-msg-queue-proof.hpp"
#include <ton/ton-tl.hpp>
namespace ton { namespace ton {
namespace validator { namespace validator {
@ -54,7 +56,14 @@ struct AsyncSerializerState {
}; };
struct CollationStats { struct CollationStats {
BlockIdExt block_id;
td::Bits256 collated_data_hash = td::Bits256::zero();
CatchainSeqno cc_seqno = 0;
double collated_at = -1.0;
td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0;
int attempt = 0;
td::Bits256 collator_node_id = td::Bits256::zero();
td::Bits256 validator_id = td::Bits256::zero();
td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0;
int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0;
std::string limits_log; std::string limits_log;
@ -62,15 +71,51 @@ struct CollationStats {
td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_filtered = 0;
td::uint32 ext_msgs_accepted = 0; td::uint32 ext_msgs_accepted = 0;
td::uint32 ext_msgs_rejected = 0; td::uint32 ext_msgs_rejected = 0;
double work_time = 0.0, cpu_work_time = 0.0; double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0;
td::uint32 serialized_size = 0, serialized_size_no_collated_data = 0; std::string time_stats;
tl_object_ptr<ton_api::validatorSession_collationStats> tl() const { tl_object_ptr<ton_api::validatorStats_collatedBlock> tl() const {
return create_tl_object<ton_api::validatorSession_collationStats>( int flags = (collator_node_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::COLLATOR_NODE_ID_MASK) |
actual_bytes, actual_collated_data_bytes, estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, (validator_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::VALIDATOR_ID_MASK);
cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, limits_log, ext_msgs_total, ext_msgs_filtered, return create_tl_object<ton_api::validatorStats_collatedBlock>(
ext_msgs_accepted, ext_msgs_rejected, work_time, cpu_work_time, serialized_size, flags, create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes,
serialized_size_no_collated_data); actual_collated_data_bytes, attempt, collator_node_id, validator_id, total_time, work_time, cpu_work_time,
time_stats,
create_tl_object<ton_api::validatorStats_blockLimitsStatus>(estimated_bytes, gas, lt_delta,
estimated_collated_data_bytes, cat_bytes, cat_gas,
cat_lt_delta, cat_collated_data_bytes, limits_log),
create_tl_object<ton_api::validatorStats_extMsgsStats>(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted,
ext_msgs_rejected));
}
};
struct ValidationStats {
BlockIdExt block_id;
td::Bits256 collated_data_hash = td::Bits256::zero();
double validated_at = -1.0;
bool valid = false;
std::string comment;
td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0;
double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0;
tl_object_ptr<ton_api::validatorStats_validatedBlock> tl() const {
return create_tl_object<ton_api::validatorStats_validatedBlock>(
create_tl_block_id(block_id), collated_data_hash, validated_at, valid, comment, actual_bytes,
actual_collated_data_bytes, total_time, work_time, cpu_work_time);
}
};
struct CollatorNodeResponseStats {
td::Bits256 collator_node_id = td::Bits256::zero();
td::Bits256 validator_id = td::Bits256::zero();
double timestamp = -1.0;
BlockIdExt block_id, original_block_id;
td::Bits256 collated_data_hash = td::Bits256::zero();
tl_object_ptr<ton_api::validatorStats_collatorNodeResponse> tl() const {
return create_tl_object<ton_api::validatorStats_collatorNodeResponse>(
collator_node_id, validator_id, timestamp, create_tl_block_id(block_id), create_tl_block_id(original_block_id),
collated_data_hash);;
} }
}; };
@ -198,9 +243,12 @@ class ValidatorManager : public ValidatorManagerInterface {
virtual void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) = 0; virtual void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) = 0;
virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; virtual void log_validator_session_stats(validatorsession::ValidatorSessionStats stats) {
virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; }
virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) = 0; virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) {
}
virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) {
}
virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise<ConstBlockHandle> promise) = 0; virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise<ConstBlockHandle> promise) = 0;
virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise<td::Ref<BlockData>> promise) = 0; virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise<td::Ref<BlockData>> promise) = 0;
@ -220,9 +268,11 @@ class ValidatorManager : public ValidatorManagerInterface {
virtual void add_lite_query_stats(int lite_query_id) { virtual void add_lite_query_stats(int lite_query_id) {
} }
virtual void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) { virtual void log_collate_query_stats(CollationStats stats) {
} }
virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { virtual void log_validate_query_stats(ValidationStats stats) {
}
virtual void log_collator_node_response_stats(CollatorNodeResponseStats stats) {
} }
virtual void add_persistent_state_description(td::Ref<PersistentStateDescription> desc) = 0; virtual void add_persistent_state_description(td::Ref<PersistentStateDescription> desc) = 0;

3
validator/manager-disk.cpp
View file

@ -129,7 +129,8 @@ void ValidatorManagerImpl::sync_complete(td::Promise<td::Unit> promise) {
Ed25519_PublicKey created_by{td::Bits256::zero()}; Ed25519_PublicKey created_by{td::Bits256::zero()};
td::as<td::uint32>(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); td::as<td::uint32>(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8);
run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref<CollatorOptions>{true}, run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref<CollatorOptions>{true},
actor_id(this), td::Timestamp::in(10.0), std::move(P), td::CancellationToken{}, 0); actor_id(this), td::Timestamp::in(10.0), std::move(P), adnl::AdnlNodeIdShort::zero(),
td::CancellationToken{}, 0);
} }
void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector<BlockIdExt> prev, BlockIdExt last, void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector<BlockIdExt> prev, BlockIdExt last,

9
validator/manager-disk.hpp
View file

@ -403,15 +403,6 @@ class ValidatorManagerImpl : public ValidatorManager {
void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) override { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) override {
UNREACHABLE(); UNREACHABLE();
} }
void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override {
UNREACHABLE();
}
void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override {
UNREACHABLE();
}
void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override {
UNREACHABLE();
}
void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint64> promise) override { void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint64> promise) override {
if (queue_size_counter_.empty()) { if (queue_size_counter_.empty()) {
queue_size_counter_ = td::actor::create_actor<QueueSizeCounter>("queuesizecounter", td::Ref<MasterchainState>{}, queue_size_counter_ = td::actor::create_actor<QueueSizeCounter>("queuesizecounter", td::Ref<MasterchainState>{},

9
validator/manager-hardfork.hpp
View file

@ -468,15 +468,6 @@ class ValidatorManagerImpl : public ValidatorManager {
void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) override { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) override {
UNREACHABLE(); UNREACHABLE();
} }
void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override {
UNREACHABLE();
}
void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override {
UNREACHABLE();
}
void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override {
UNREACHABLE();
}
void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint64> promise) override { void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint64> promise) override {
if (queue_size_counter_.empty()) { if (queue_size_counter_.empty()) {
queue_size_counter_ = td::actor::create_actor<QueueSizeCounter>("queuesizecounter", td::Ref<MasterchainState>{}, queue_size_counter_ = td::actor::create_actor<QueueSizeCounter>("queuesizecounter", td::Ref<MasterchainState>{},

126
validator/manager.cpp
View file

@ -3107,61 +3107,20 @@ void ValidatorManagerImpl::wait_shard_client_state(BlockSeqno seqno, td::Timesta
shard_client_waiters_[seqno].waiting_.emplace_back(timeout, 0, std::move(promise)); shard_client_waiters_[seqno].waiting_.emplace_back(timeout, 0, std::move(promise));
} }
void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, void ValidatorManagerImpl::log_validator_session_stats(validatorsession::ValidatorSessionStats stats) {
validatorsession::ValidatorSessionStats stats) {
std::string fname = opts_->get_session_logs_file(); std::string fname = opts_->get_session_logs_file();
if (fname.empty()) { if (fname.empty()) {
return; return;
} }
stats.fix_block_ids();
std::vector<tl_object_ptr<ton_api::validatorSession_statsRound>> rounds; auto obj = stats.tl();
for (const auto &round : stats.rounds) {
std::vector<tl_object_ptr<ton_api::validatorSession_statsProducer>> producers;
for (const auto &producer : round.producers) {
BlockIdExt cur_block_id{block_id.id, producer.root_hash, producer.file_hash};
auto it = recorded_block_stats_.find(cur_block_id);
tl_object_ptr<ton_api::validatorSession_collationStats> collation_stats;
if (it != recorded_block_stats_.end() && it->second.collator_stats_) {
auto &stats = it->second.collator_stats_.value();
collation_stats = stats.tl();
}
std::string approvers, signers;
for (bool x : producer.approvers) {
approvers += (x ? '1' : '0');
}
for (bool x : producer.signers) {
signers += (x ? '1' : '0');
}
producers.push_back(create_tl_object<ton_api::validatorSession_statsProducer>(
producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.root_hash,
producer.file_hash, producer.comment, producer.block_timestamp, producer.is_accepted, producer.is_ours,
producer.got_submit_at, producer.collation_time, producer.collated_at, producer.collation_cached,
it == recorded_block_stats_.end() ? -1.0 : it->second.collator_work_time_,
it == recorded_block_stats_.end() ? -1.0 : it->second.collator_cpu_work_time_, std::move(collation_stats),
producer.validation_time, producer.validated_at, producer.validation_cached,
it == recorded_block_stats_.end() ? -1.0 : it->second.validator_work_time_,
it == recorded_block_stats_.end() ? -1.0 : it->second.validator_cpu_work_time_, producer.gen_utime,
producer.approved_weight, producer.approved_33pct_at, producer.approved_66pct_at, std::move(approvers),
producer.signed_weight, producer.signed_33pct_at, producer.signed_66pct_at, std::move(signers),
producer.serialize_time, producer.deserialize_time, producer.serialized_size));
}
rounds.push_back(create_tl_object<ton_api::validatorSession_statsRound>(round.timestamp, std::move(producers)));
}
auto obj = create_tl_object<ton_api::validatorSession_stats>(
stats.success, create_tl_block_id(block_id), stats.timestamp, stats.self.bits256_value(), stats.session_id,
stats.cc_seqno, stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures,
stats.signatures_weight, stats.approve_signatures, stats.approve_signatures_weight, stats.first_round,
std::move(rounds));
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false); auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
std::ofstream file; std::ofstream file;
file.open(fname, std::ios_base::app); file.open(fname, std::ios_base::app);
file << s << "\n"; file << s << "\n";
file.close(); file.close();
LOG(INFO) << "Writing validator session stats for " << stats.block_id.to_str();
LOG(INFO) << "Writing validator session stats for " << block_id.id.to_str();
} }
void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) {
@ -3169,22 +3128,13 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa
if (fname.empty()) { if (fname.empty()) {
return; return;
} }
std::vector<tl_object_ptr<ton_api::validatorSession_newValidatorGroupStats_node>> nodes; auto obj = stats.tl();
for (const auto &node : stats.nodes) {
nodes.push_back(
create_tl_object<ton_api::validatorSession_newValidatorGroupStats_node>(node.id.bits256_value(), node.weight));
}
auto obj = create_tl_object<ton_api::validatorSession_newValidatorGroupStats>(
stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.last_key_block_seqno,
stats.timestamp, stats.self_idx, std::move(nodes));
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false); auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
std::ofstream file; std::ofstream file;
file.open(fname, std::ios_base::app); file.open(fname, std::ios_base::app);
file << s << "\n"; file << s << "\n";
file.close(); file.close();
LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str() LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str()
<< " cc_seqno=" << stats.cc_seqno; << " cc_seqno=" << stats.cc_seqno;
} }
@ -3194,21 +3144,13 @@ void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndVa
if (fname.empty()) { if (fname.empty()) {
return; return;
} }
std::vector<tl_object_ptr<ton_api::validatorSession_endValidatorGroupStats_node>> nodes; auto obj = stats.tl();
for (const auto &node : stats.nodes) {
nodes.push_back(create_tl_object<ton_api::validatorSession_endValidatorGroupStats_node>(node.id.bits256_value(),
node.catchain_blocks));
}
auto obj = create_tl_object<ton_api::validatorSession_endValidatorGroupStats>(stats.session_id, stats.timestamp,
std::move(nodes));
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false); auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
std::ofstream file; std::ofstream file;
file.open(fname, std::ios_base::app); file.open(fname, std::ios_base::app);
file << s << "\n"; file << s << "\n";
file.close(); file.close();
LOG(INFO) << "Writing end validator group stats for " << stats.session_id; LOG(INFO) << "Writing end validator group stats for " << stats.session_id;
} }
@ -3657,45 +3599,49 @@ td::actor::ActorOwn<ValidatorManagerInterface> ValidatorManagerFactory::create(
rldp, overlays); rldp, overlays);
} }
void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, CollationStats stats) { void ValidatorManagerImpl::log_collate_query_stats(CollationStats stats) {
auto &record = new_block_stats_record(block_id);
record.collator_work_time_ = stats.work_time;
record.collator_cpu_work_time_ = stats.cpu_work_time;
record.collator_stats_ = std::move(stats);
std::string fname = opts_->get_session_logs_file(); std::string fname = opts_->get_session_logs_file();
if (fname.empty()) { if (fname.empty()) {
return; return;
} }
auto obj = stats.tl();
auto obj = create_tl_object<ton_api::validatorSession_statsCollatedBlock>(td::Clocks::system(),
create_tl_block_id(block_id), stats.tl());
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false); auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
std::ofstream file; std::ofstream file;
file.open(fname, std::ios_base::app); file.open(fname, std::ios_base::app);
file << s << "\n"; file << s << "\n";
file.close(); file.close();
LOG(DEBUG) << "Writing collation stats stats for " << stats.block_id.to_str();
LOG(DEBUG) << "Writing collation stats stats for " << block_id.id.to_str();
} }
void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { void ValidatorManagerImpl::log_validate_query_stats(ValidationStats stats) {
auto &record = new_block_stats_record(block_id); std::string fname = opts_->get_session_logs_file();
record.validator_work_time_ = work_time; if (fname.empty()) {
record.validator_cpu_work_time_ = cpu_work_time; return;
}
ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) {
if (!recorded_block_stats_.count(block_id)) {
recorded_block_stats_lru_.push(block_id);
if (recorded_block_stats_lru_.size() > 4096) {
recorded_block_stats_.erase(recorded_block_stats_lru_.front());
recorded_block_stats_lru_.pop();
}
} }
return recorded_block_stats_[block_id]; auto obj = stats.tl();
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
std::ofstream file;
file.open(fname, std::ios_base::app);
file << s << "\n";
file.close();
LOG(DEBUG) << "Writing validation stats stats for " << stats.block_id.to_str();
}
void ValidatorManagerImpl::log_collator_node_response_stats(CollatorNodeResponseStats stats) {
std::string fname = opts_->get_session_logs_file();
if (fname.empty()) {
return;
}
auto obj = stats.tl();
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
std::ofstream file;
file.open(fname, std::ios_base::app);
file << s << "\n";
file.close();
LOG(DEBUG) << "Writing collator node response stats stats for " << stats.block_id.to_str();
} }
size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) {

18
validator/manager.hpp
View file

@ -630,7 +630,7 @@ class ValidatorManagerImpl : public ValidatorManager {
void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) override; void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) override;
void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; void log_validator_session_stats(validatorsession::ValidatorSessionStats stats) override;
void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override;
void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override; void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override;
@ -775,19 +775,9 @@ class ValidatorManagerImpl : public ValidatorManager {
td::actor::ActorOwn<CandidatesBuffer> candidates_buffer_; td::actor::ActorOwn<CandidatesBuffer> candidates_buffer_;
struct RecordedBlockStats { void log_collate_query_stats(CollationStats stats) override;
double collator_work_time_ = -1.0; void log_validate_query_stats(ValidationStats stats) override;
double collator_cpu_work_time_ = -1.0; void log_collator_node_response_stats(CollatorNodeResponseStats stats) override;
td::optional<CollationStats> collator_stats_;
double validator_work_time_ = -1.0;
double validator_cpu_work_time_ = -1.0;
};
std::map<BlockIdExt, RecordedBlockStats> recorded_block_stats_;
std::queue<BlockIdExt> recorded_block_stats_lru_;
void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) override;
void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override;
RecordedBlockStats &new_block_stats_record(BlockIdExt block_id);
std::map<PublicKeyHash, td::actor::ActorOwn<ValidatorTelemetry>> validator_telemetry_; std::map<PublicKeyHash, td::actor::ActorOwn<ValidatorTelemetry>> validator_telemetry_;

55
validator/validator-group.cpp
View file

@ -36,9 +36,8 @@ static bool need_send_candidate_broadcast(const validatorsession::BlockSourceInf
!is_masterchain; !is_masterchain;
} }
void ValidatorGroup::generate_block_candidate( void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo source_info,
validatorsession::BlockSourceInfo source_info, td::Promise<GeneratedCandidate> promise) {
td::Promise<validatorsession::ValidatorSession::GeneratedCandidate> promise) {
td::uint32 round_id = source_info.priority.round; td::uint32 round_id = source_info.priority.round;
if (round_id > last_known_round_id_) { if (round_id > last_known_round_id_) {
last_known_round_id_ = round_id; last_known_round_id_ = round_id;
@ -49,20 +48,21 @@ void ValidatorGroup::generate_block_candidate(
} }
if (cached_collated_block_) { if (cached_collated_block_) {
if (cached_collated_block_->result) { if (cached_collated_block_->result) {
promise.set_value({cached_collated_block_->result.value().clone(), true}); auto res = cached_collated_block_->result.value().clone();
res.is_cached = true;
promise.set_value(std::move(res));
} else { } else {
cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { cached_collated_block_->promises.push_back(promise.wrap([](GeneratedCandidate &&res) {
return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), true}; res.is_cached = true;
return std::move(res);
})); }));
} }
return; return;
} }
cached_collated_block_ = std::make_shared<CachedCollatedBlock>(); cached_collated_block_ = std::make_shared<CachedCollatedBlock>();
cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { cached_collated_block_->promises.push_back(std::move(promise));
return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), false}; td::Promise<GeneratedCandidate> P = [SelfId = actor_id(this), cache = cached_collated_block_,
})); source_info](td::Result<GeneratedCandidate> R) {
td::Promise<BlockCandidate> P = [SelfId = actor_id(this), cache = cached_collated_block_,
source_info](td::Result<BlockCandidate> R) {
td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache), td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache),
std::move(R)); std::move(R));
}; };
@ -75,7 +75,7 @@ void ValidatorGroup::generate_block_candidate(
void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info,
std::shared_ptr<CachedCollatedBlock> cache, std::shared_ptr<CachedCollatedBlock> cache,
td::Result<BlockCandidate> R) { td::Result<GeneratedCandidate> R) {
if (R.is_error()) { if (R.is_error()) {
for (auto &p : cache->promises) { for (auto &p : cache->promises) {
p.set_error(R.error().clone()); p.set_error(R.error().clone());
@ -84,12 +84,12 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo
cached_collated_block_ = nullptr; cached_collated_block_ = nullptr;
} }
} else { } else {
auto candidate = R.move_as_ok(); auto c = R.move_as_ok();
add_available_block_candidate(candidate.pubkey.as_bits256(), candidate.id, candidate.collated_file_hash); add_available_block_candidate(c.candidate.pubkey.as_bits256(), c.candidate.id, c.candidate.collated_file_hash);
if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) { if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) {
send_block_candidate_broadcast(candidate.id, candidate.data.clone()); send_block_candidate_broadcast(c.candidate.id, c.candidate.data.clone());
} }
cache->result = std::move(candidate); cache->result = std::move(c);
for (auto &p : cache->promises) { for (auto &p : cache->promises) {
p.set_value(cache->result.value().clone()); p.set_value(cache->result.value().clone());
} }
@ -186,7 +186,8 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so
} }
auto next_block_id = create_next_block_id(root_hash, file_hash); auto next_block_id = create_next_block_id(root_hash, file_hash);
LOG(WARNING) << "Accepted block " << next_block_id; LOG(WARNING) << "Accepted block " << next_block_id;
td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, std::move(stats)); stats.block_id = next_block_id;
td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(stats));
auto block = auto block =
block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref<BlockData>{}; block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref<BlockData>{};
@ -293,7 +294,7 @@ std::unique_ptr<validatorsession::ValidatorSession::Callback> ValidatorGroup::ma
std::move(candidate), std::move(P)); std::move(candidate), std::move(P));
} }
void on_generate_slot(validatorsession::BlockSourceInfo source_info, void on_generate_slot(validatorsession::BlockSourceInfo source_info,
td::Promise<validatorsession::ValidatorSession::GeneratedCandidate> promise) override { td::Promise<GeneratedCandidate> promise) override {
td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, std::move(source_info), td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, std::move(source_info),
std::move(promise)); std::move(promise));
} }
@ -397,8 +398,8 @@ void ValidatorGroup::start(std::vector<BlockIdExt> prev, BlockIdExt min_masterch
for (auto &p : postponed_accept_) { for (auto &p : postponed_accept_) {
auto next_block_id = create_next_block_id(p.root_hash, p.file_hash); auto next_block_id = create_next_block_id(p.root_hash, p.file_hash);
td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, p.stats.block_id = next_block_id;
std::move(p.stats)); td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(p.stats));
auto block = auto block =
p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref<BlockData>{}; p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref<BlockData>{};
@ -408,12 +409,11 @@ void ValidatorGroup::start(std::vector<BlockIdExt> prev, BlockIdExt min_masterch
} }
postponed_accept_.clear(); postponed_accept_.clear();
validatorsession::NewValidatorGroupStats stats; validatorsession::NewValidatorGroupStats stats{.session_id = session_id_,
stats.session_id = session_id_; .shard = shard_,
stats.shard = shard_; .cc_seqno = validator_set_->get_catchain_seqno(),
stats.cc_seqno = validator_set_->get_catchain_seqno(); .last_key_block_seqno = last_key_block_seqno_,
stats.last_key_block_seqno = last_key_block_seqno_; .started_at = td::Clocks::system()};
stats.timestamp = td::Clocks::system();
td::uint32 idx = 0; td::uint32 idx = 0;
for (const auto &node : validator_set_->export_vector()) { for (const auto &node : validator_set_->export_vector()) {
PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id();
@ -441,7 +441,8 @@ void ValidatorGroup::destroy() {
return; return;
} }
stats.cc_seqno = cc_seqno; stats.cc_seqno = cc_seqno;
td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, stats.block_id = block_id;
td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats,
std::move(stats)); std::move(stats));
}); });
td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats,

12
validator/validator-group.hpp
View file

@ -35,8 +35,7 @@ class ValidatorManager;
class ValidatorGroup : public td::actor::Actor { class ValidatorGroup : public td::actor::Actor {
public: public:
void generate_block_candidate(validatorsession::BlockSourceInfo source_info, void generate_block_candidate(validatorsession::BlockSourceInfo source_info, td::Promise<GeneratedCandidate> promise);
td::Promise<validatorsession::ValidatorSession::GeneratedCandidate> promise);
void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block,
td::Promise<std::pair<UnixTime, bool>> promise); td::Promise<std::pair<UnixTime, bool>> promise);
void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash, void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash,
@ -146,14 +145,14 @@ class ValidatorGroup : public td::actor::Actor {
bool monitoring_shard_ = true; bool monitoring_shard_ = true;
struct CachedCollatedBlock { struct CachedCollatedBlock {
td::optional<BlockCandidate> result; td::optional<GeneratedCandidate> result;
std::vector<td::Promise<BlockCandidate>> promises; std::vector<td::Promise<GeneratedCandidate>> promises;
}; };
std::shared_ptr<CachedCollatedBlock> cached_collated_block_; std::shared_ptr<CachedCollatedBlock> cached_collated_block_;
td::CancellationTokenSource cancellation_token_source_; td::CancellationTokenSource cancellation_token_source_;
void generated_block_candidate(validatorsession::BlockSourceInfo source_info, void generated_block_candidate(validatorsession::BlockSourceInfo source_info,
std::shared_ptr<CachedCollatedBlock> cache, td::Result<BlockCandidate> R); std::shared_ptr<CachedCollatedBlock> cache, td::Result<GeneratedCandidate> R);
using CacheKey = std::tuple<td::Bits256, BlockIdExt, FileHash, FileHash>; using CacheKey = std::tuple<td::Bits256, BlockIdExt, FileHash, FileHash>;
std::map<CacheKey, UnixTime> approved_candidates_cache_; std::map<CacheKey, UnixTime> approved_candidates_cache_;
@ -165,8 +164,7 @@ class ValidatorGroup : public td::actor::Actor {
} }
void get_validator_group_info_for_litequery_cont( void get_validator_group_info_for_litequery_cont(
td::uint32 expected_round, td::uint32 expected_round, std::vector<tl_object_ptr<lite_api::liteServer_nonfinal_candidateInfo>> candidates,
std::vector<tl_object_ptr<lite_api::liteServer_nonfinal_candidateInfo>> candidates,
td::Promise<tl_object_ptr<lite_api::liteServer_nonfinal_validatorGroupInfo>> promise); td::Promise<tl_object_ptr<lite_api::liteServer_nonfinal_validatorGroupInfo>> promise);
std::set<std::tuple<td::Bits256, BlockIdExt, FileHash>> available_block_candidates_; // source, id, collated hash std::set<std::tuple<td::Bits256, BlockIdExt, FileHash>> available_block_candidates_; // source, id, collated hash