mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
Rework session stats
This commit is contained in:
SpyCheese
parent
392cf64758
commit
71342bdcd4
24 changed files with 436 additions and 334 deletions
|
|
@ -33,13 +33,16 @@ void CollationManager::start_up() {
|
|||
|
||||
void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id,
|
||||
std::vector<BlockIdExt> prev, Ed25519_PublicKey creator,
|
||||
BlockCandidatePriority priority,
|
||||
td::Ref<ValidatorSet> validator_set, td::uint64 max_answer_size,
|
||||
td::CancellationToken cancellation_token, td::Promise<BlockCandidate> promise) {
|
||||
BlockCandidatePriority priority, td::Ref<ValidatorSet> validator_set,
|
||||
td::uint64 max_answer_size, td::CancellationToken cancellation_token,
|
||||
td::Promise<GeneratedCandidate> promise) {
|
||||
if (shard.is_masterchain()) {
|
||||
run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set),
|
||||
opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise),
|
||||
std::move(cancellation_token), 0);
|
||||
run_collate_query(
|
||||
shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set),
|
||||
opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) {
|
||||
return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true};
|
||||
}),
|
||||
adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0);
|
||||
return;
|
||||
}
|
||||
collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set),
|
||||
|
|
@ -48,10 +51,9 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha
|
|||
|
||||
void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id,
|
||||
std::vector<BlockIdExt> prev, Ed25519_PublicKey creator,
|
||||
BlockCandidatePriority priority,
|
||||
td::Ref<ValidatorSet> validator_set, td::uint64 max_answer_size,
|
||||
td::CancellationToken cancellation_token,
|
||||
td::Promise<BlockCandidate> promise, td::Timestamp timeout) {
|
||||
BlockCandidatePriority priority, td::Ref<ValidatorSet> validator_set,
|
||||
td::uint64 max_answer_size, td::CancellationToken cancellation_token,
|
||||
td::Promise<GeneratedCandidate> promise, td::Timestamp timeout) {
|
||||
TRY_STATUS_PROMISE(promise, cancellation_token.check());
|
||||
ShardInfo* s = select_shard_info(shard);
|
||||
if (s == nullptr) {
|
||||
|
|
@ -104,9 +106,12 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas
|
|||
}
|
||||
|
||||
if (selected_collator.is_zero() && s->self_collate) {
|
||||
run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set),
|
||||
opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise),
|
||||
std::move(cancellation_token), 0);
|
||||
run_collate_query(
|
||||
shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set),
|
||||
opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) {
|
||||
return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true};
|
||||
}),
|
||||
adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -117,10 +122,13 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas
|
|||
next_block_id.seqno = std::max(next_block_id.seqno, p.seqno() + 1);
|
||||
}
|
||||
|
||||
promise = [=, SelfId = actor_id(this), promise = std::move(promise),
|
||||
retry_at = td::Timestamp::in(0.5)](td::Result<BlockCandidate> R) mutable {
|
||||
td::Promise<BlockCandidate> P = [=, SelfId = actor_id(this), promise = std::move(promise),
|
||||
retry_at = td::Timestamp::in(0.5)](td::Result<BlockCandidate> R) mutable {
|
||||
if (R.is_ok()) {
|
||||
promise.set_value(R.move_as_ok());
|
||||
promise.set_value(GeneratedCandidate{.candidate = R.move_as_ok(),
|
||||
.is_cached = false,
|
||||
.self_collated = false,
|
||||
.collator_node_id = selected_collator.bits256_value()});
|
||||
return;
|
||||
}
|
||||
if (!selected_collator.is_zero()) {
|
||||
|
|
@ -142,7 +150,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas
|
|||
};
|
||||
|
||||
if (selected_collator.is_zero()) {
|
||||
promise.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node"));
|
||||
P.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node"));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -152,32 +160,32 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas
|
|||
LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "("
|
||||
<< selected_collator << ")";
|
||||
|
||||
td::Promise<td::BufferSlice> P = [=, SelfId = actor_id(this), promise = std::move(promise),
|
||||
timer = td::Timer()](td::Result<td::BufferSlice> R) mutable {
|
||||
TRY_RESULT_PROMISE_PREFIX(promise, data, std::move(R), "rldp query failed: ");
|
||||
td::Promise<td::BufferSlice> P2 = [=, SelfId = actor_id(this), P = std::move(P),
|
||||
timer = td::Timer()](td::Result<td::BufferSlice> R) mutable {
|
||||
TRY_RESULT_PROMISE_PREFIX(P, data, std::move(R), "rldp query failed: ");
|
||||
auto r_error = fetch_tl_object<ton_api::collatorNode_error>(data, true);
|
||||
if (r_error.is_ok()) {
|
||||
auto error = r_error.move_as_ok();
|
||||
promise.set_error(td::Status::Error(error->code_, error->message_));
|
||||
P.set_error(td::Status::Error(error->code_, error->message_));
|
||||
return;
|
||||
}
|
||||
TRY_RESULT_PROMISE(promise, f, fetch_tl_object<ton_api::collatorNode_Candidate>(data, true));
|
||||
TRY_RESULT_PROMISE(promise, candidate,
|
||||
TRY_RESULT_PROMISE(P, f, fetch_tl_object<ton_api::collatorNode_Candidate>(data, true));
|
||||
TRY_RESULT_PROMISE(P, candidate,
|
||||
CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast<int>(max_answer_size)));
|
||||
if (candidate.pubkey.as_bits256() != creator.as_bits256()) {
|
||||
promise.set_error(td::Status::Error("collate query: block candidate source mismatch"));
|
||||
P.set_error(td::Status::Error("collate query: block candidate source mismatch"));
|
||||
return;
|
||||
}
|
||||
if (candidate.id.id != next_block_id) {
|
||||
promise.set_error(td::Status::Error("collate query: block id mismatch"));
|
||||
P.set_error(td::Status::Error("collate query: block id mismatch"));
|
||||
return;
|
||||
}
|
||||
LOG(INFO) << "got collated block " << next_block_id.to_str() << " from #" << selected_idx << " ("
|
||||
<< selected_collator << ") in " << timer.elapsed() << "s";
|
||||
promise.set_result(std::move(candidate));
|
||||
P.set_result(std::move(candidate));
|
||||
};
|
||||
td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", std::move(P),
|
||||
timeout, std::move(query), max_answer_size);
|
||||
td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery",
|
||||
std::move(P2), timeout, std::move(query), max_answer_size);
|
||||
}
|
||||
|
||||
void CollationManager::update_options(td::Ref<ValidatorManagerOptions> opts) {
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ class CollationManager : public td::actor::Actor {
|
|||
void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev,
|
||||
Ed25519_PublicKey creator, BlockCandidatePriority priority,
|
||||
td::Ref<ValidatorSet> validator_set, td::uint64 max_answer_size,
|
||||
td::CancellationToken cancellation_token, td::Promise<BlockCandidate> promise);
|
||||
td::CancellationToken cancellation_token, td::Promise<GeneratedCandidate> promise);
|
||||
|
||||
void update_options(td::Ref<ValidatorManagerOptions> opts);
|
||||
|
||||
|
|
@ -55,7 +55,7 @@ class CollationManager : public td::actor::Actor {
|
|||
void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev,
|
||||
Ed25519_PublicKey creator, BlockCandidatePriority priority,
|
||||
td::Ref<ValidatorSet> validator_set, td::uint64 max_answer_size,
|
||||
td::CancellationToken cancellation_token, td::Promise<BlockCandidate> promise,
|
||||
td::CancellationToken cancellation_token, td::Promise<GeneratedCandidate> promise,
|
||||
td::Timestamp timeout);
|
||||
|
||||
void update_collators_list(const CollatorsList& collators_list);
|
||||
|
|
|
|||
|
|
@ -391,12 +391,24 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data
|
|||
promise.set_result(serialize_tl_object(serialize_candidate(R.move_as_ok(), true), true));
|
||||
}
|
||||
};
|
||||
new_promise = [new_promise = std::move(new_promise), creator,
|
||||
new_promise = [new_promise = std::move(new_promise), creator, local_id = local_id_,
|
||||
manager = manager_](td::Result<BlockCandidate> R) mutable {
|
||||
TRY_RESULT_PROMISE(new_promise, block, std::move(R));
|
||||
|
||||
CollatorNodeResponseStats stats;
|
||||
stats.collator_node_id = local_id.bits256_value();
|
||||
stats.validator_id = creator.as_bits256();
|
||||
stats.original_block_id = block.id;
|
||||
stats.collated_data_hash = block.collated_file_hash;
|
||||
|
||||
CatchainSeqno cc_seqno;
|
||||
td::uint32 val_set_hash;
|
||||
block = change_creator(std::move(block), creator, cc_seqno, val_set_hash);
|
||||
|
||||
stats.block_id = block.id;
|
||||
stats.timestamp = td::Clocks::system();
|
||||
td::actor::send_closure(manager, &ValidatorManager::log_collator_node_response_stats, std::move(stats));
|
||||
|
||||
td::Promise<td::Unit> P =
|
||||
new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); });
|
||||
td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno,
|
||||
|
|
@ -537,7 +549,7 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std
|
|||
};
|
||||
td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R));
|
||||
},
|
||||
cache_entry->cancellation_token_source.get_cancellation_token(),
|
||||
local_id_, cache_entry->cancellation_token_source.get_cancellation_token(),
|
||||
CollateMode::skip_store_candidate | CollateMode::from_collator_node);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -89,7 +89,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc
|
|||
Ed25519_PublicKey creator, td::Ref<ValidatorSet> validator_set,
|
||||
td::Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager,
|
||||
td::Timestamp timeout, td::Promise<BlockCandidate> promise,
|
||||
td::CancellationToken cancellation_token, unsigned mode, int attempt_idx = 0);
|
||||
adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode,
|
||||
int attempt_idx = 0);
|
||||
void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector<BlockIdExt> prev,
|
||||
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
|
||||
td::Promise<BlockCandidate> promise);
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ class Collator final : public td::actor::Actor {
|
|||
td::Timestamp timeout;
|
||||
td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_;
|
||||
td::Promise<BlockCandidate> main_promise;
|
||||
adnl::AdnlNodeIdShort collator_node_id_ = adnl::AdnlNodeIdShort::zero();
|
||||
unsigned mode_ = 0;
|
||||
int attempt_idx_;
|
||||
bool allow_repeat_collation_ = false;
|
||||
|
|
@ -97,7 +98,8 @@ class Collator final : public td::actor::Actor {
|
|||
Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev,
|
||||
Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id, Ref<CollatorOptions> collator_opts,
|
||||
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout, td::Promise<BlockCandidate> promise,
|
||||
td::CancellationToken cancellation_token, unsigned mode, int attempt_idx);
|
||||
adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode,
|
||||
int attempt_idx);
|
||||
~Collator() override = default;
|
||||
bool is_busy() const {
|
||||
return busy_;
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ static inline bool dbg(int c) {
|
|||
* @param manager The ActorId of the ValidatorManager.
|
||||
* @param timeout The timeout for the collator.
|
||||
* @param promise The promise to return the result.
|
||||
* @param collator_node_id ADNL id of the collator node that generates the block (zero if it's not a collator node)
|
||||
* @param cancellation_token Token to cancel collation.
|
||||
* @param mode +1 - skip storing candidate to disk, +2 - called from CollatorNode.
|
||||
* @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps.
|
||||
|
|
@ -84,8 +85,8 @@ static inline bool dbg(int c) {
|
|||
Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id,
|
||||
std::vector<BlockIdExt> prev, td::Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id,
|
||||
Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager,
|
||||
td::Timestamp timeout, td::Promise<BlockCandidate> promise, td::CancellationToken cancellation_token,
|
||||
unsigned mode, int attempt_idx)
|
||||
td::Timestamp timeout, td::Promise<BlockCandidate> promise, adnl::AdnlNodeIdShort collator_node_id,
|
||||
td::CancellationToken cancellation_token, unsigned mode, int attempt_idx)
|
||||
: shard_(shard)
|
||||
, is_hardfork_(is_hardfork)
|
||||
, min_mc_block_id{min_masterchain_block_id}
|
||||
|
|
@ -100,6 +101,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha
|
|||
, soft_timeout_(td::Timestamp::at(timeout.at() - 3.0))
|
||||
, medium_timeout_(td::Timestamp::at(timeout.at() - 1.5))
|
||||
, main_promise(std::move(promise))
|
||||
, collator_node_id_(collator_node_id)
|
||||
, mode_(mode)
|
||||
, attempt_idx_(attempt_idx)
|
||||
, perf_timer_("collate", 0.1,
|
||||
|
|
@ -374,8 +376,8 @@ bool Collator::fatal_error(td::Status error) {
|
|||
!is_hardfork_ && !timeout.is_in_past()) {
|
||||
LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")";
|
||||
run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager,
|
||||
td::Timestamp::in(10.0), std::move(main_promise), std::move(cancellation_token_), mode_,
|
||||
attempt_idx_ + 1);
|
||||
td::Timestamp::in(10.0), std::move(main_promise), collator_node_id_,
|
||||
std::move(cancellation_token_), mode_, attempt_idx_ + 1);
|
||||
} else {
|
||||
LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error;
|
||||
LOG(INFO) << perf_log_;
|
||||
|
|
@ -5927,8 +5929,15 @@ bool Collator::create_block_candidate() {
|
|||
double work_time = work_timer_.elapsed();
|
||||
double cpu_work_time = cpu_work_timer_.elapsed();
|
||||
LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s";
|
||||
stats_.block_id = block_candidate->id;
|
||||
stats_.collated_data_hash = block_candidate->collated_file_hash;
|
||||
stats_.cc_seqno = validator_set_->get_catchain_seqno();
|
||||
stats_.collated_at = td::Clocks::system();
|
||||
stats_.actual_bytes = block_candidate->data.size();
|
||||
stats_.actual_collated_data_bytes = block_candidate->collated_data.size();
|
||||
stats_.attempt = attempt_idx_;
|
||||
stats_.collator_node_id = collator_node_id_.bits256_value();
|
||||
stats_.validator_id = created_by_.as_bits256();
|
||||
stats_.estimated_bytes = block_limit_status_->estimate_block_size();
|
||||
stats_.gas = block_limit_status_->gas_used;
|
||||
stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt;
|
||||
|
|
@ -5938,20 +5947,12 @@ bool Collator::create_block_candidate() {
|
|||
stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt);
|
||||
stats_.cat_collated_data_bytes =
|
||||
block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes);
|
||||
stats_.total_time = perf_timer_.elapsed();
|
||||
stats_.work_time = work_time;
|
||||
stats_.cpu_work_time = cpu_work_time;
|
||||
stats_.time_stats = (PSTRING() << perf_log_);
|
||||
|
||||
// TODO: remove this later (currently needed to collect stats)
|
||||
if (mode_ & CollateMode::from_collator_node) {
|
||||
size_t d;
|
||||
stats_.serialized_size =
|
||||
validatorsession::compress_candidate_data(block_candidate->data, block_candidate->collated_data, d).ok().size();
|
||||
stats_.serialized_size_no_collated_data =
|
||||
validatorsession::compress_candidate_data(block_candidate->data, td::Slice{}, d).ok().size();
|
||||
}
|
||||
|
||||
td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id,
|
||||
std::move(stats_));
|
||||
td::actor::send_closure(manager, &ValidatorManager::log_collate_query_stats, std::move(stats_));
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -216,7 +216,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc
|
|||
Ed25519_PublicKey creator, td::Ref<ValidatorSet> validator_set,
|
||||
td::Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager,
|
||||
td::Timestamp timeout, td::Promise<BlockCandidate> promise,
|
||||
td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) {
|
||||
adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode,
|
||||
int attempt_idx) {
|
||||
BlockSeqno seqno = 0;
|
||||
for (auto& p : prev) {
|
||||
if (p.seqno() > seqno) {
|
||||
|
|
@ -227,7 +228,7 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc
|
|||
<< (attempt_idx ? "_" + td::to_string(attempt_idx) : ""),
|
||||
shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set),
|
||||
creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise),
|
||||
std::move(cancellation_token), mode, attempt_idx)
|
||||
collator_node_id, std::move(cancellation_token), mode, attempt_idx)
|
||||
.release();
|
||||
}
|
||||
|
||||
|
|
@ -240,10 +241,10 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b
|
|||
seqno = p.seqno();
|
||||
}
|
||||
}
|
||||
td::actor::create_actor<Collator>(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true,
|
||||
min_masterchain_block_id, std::move(prev), td::Ref<ValidatorSet>{},
|
||||
Ed25519_PublicKey{Bits256::zero()}, td::Ref<CollatorOptions>{true},
|
||||
std::move(manager), timeout, std::move(promise), td::CancellationToken{}, 0, 0)
|
||||
td::actor::create_actor<Collator>(
|
||||
PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id,
|
||||
std::move(prev), td::Ref<ValidatorSet>{}, Ed25519_PublicKey{Bits256::zero()}, td::Ref<CollatorOptions>{true},
|
||||
std::move(manager), timeout, std::move(promise), adnl::AdnlNodeIdShort::zero(), td::CancellationToken{}, 0, 0)
|
||||
.release();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -116,13 +116,12 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) {
|
|||
error = error_ctx() + error;
|
||||
LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error;
|
||||
if (main_promise) {
|
||||
record_stats();
|
||||
record_stats(false, error);
|
||||
errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str()
|
||||
<< " : " << error << ": data=" << block_candidate.id.file_hash.to_hex()
|
||||
<< " collated_data=" << block_candidate.collated_file_hash.to_hex());
|
||||
errorlog::ErrorLog::log_file(block_candidate.data.clone());
|
||||
errorlog::ErrorLog::log_file(block_candidate.collated_data.clone());
|
||||
LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s";
|
||||
main_promise.set_result(CandidateReject{std::move(error), std::move(reason)});
|
||||
}
|
||||
stop();
|
||||
|
|
@ -155,13 +154,12 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason)
|
|||
error = error_ctx() + error;
|
||||
LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error;
|
||||
if (main_promise) {
|
||||
record_stats();
|
||||
record_stats(false, error);
|
||||
errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str()
|
||||
<< " : " << error << ": data=" << block_candidate.id.file_hash.to_hex()
|
||||
<< " collated_data=" << block_candidate.collated_file_hash.to_hex());
|
||||
errorlog::ErrorLog::log_file(block_candidate.data.clone());
|
||||
errorlog::ErrorLog::log_file(block_candidate.collated_data.clone());
|
||||
LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s";
|
||||
main_promise.set_result(CandidateReject{std::move(error), std::move(reason)});
|
||||
}
|
||||
stop();
|
||||
|
|
@ -179,7 +177,7 @@ bool ValidateQuery::fatal_error(td::Status error) {
|
|||
error.ensure_error();
|
||||
LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string();
|
||||
if (main_promise) {
|
||||
record_stats();
|
||||
record_stats(false, error.message().str());
|
||||
auto c = error.code();
|
||||
if (c <= -667 && c >= -670) {
|
||||
errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str()
|
||||
|
|
@ -188,7 +186,6 @@ bool ValidateQuery::fatal_error(td::Status error) {
|
|||
errorlog::ErrorLog::log_file(block_candidate.data.clone());
|
||||
errorlog::ErrorLog::log_file(block_candidate.collated_data.clone());
|
||||
}
|
||||
LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s";
|
||||
main_promise(std::move(error));
|
||||
}
|
||||
stop();
|
||||
|
|
@ -238,9 +235,8 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) {
|
|||
*/
|
||||
void ValidateQuery::finish_query() {
|
||||
if (main_promise) {
|
||||
record_stats();
|
||||
record_stats(true);
|
||||
LOG(WARNING) << "validate query done";
|
||||
LOG(WARNING) << "validation took " << perf_timer_.elapsed() << " s";
|
||||
main_promise.set_result(now_);
|
||||
}
|
||||
stop();
|
||||
|
|
@ -7061,13 +7057,25 @@ void ValidateQuery::written_candidate() {
|
|||
/**
|
||||
* Sends validation work time to manager.
|
||||
*/
|
||||
void ValidateQuery::record_stats() {
|
||||
double work_time = work_timer_.elapsed();
|
||||
double cpu_work_time = cpu_work_timer_.elapsed();
|
||||
void ValidateQuery::record_stats(bool valid, std::string error_message) {
|
||||
ValidationStats stats;
|
||||
stats.block_id = id_;
|
||||
stats.collated_data_hash = block_candidate.collated_file_hash;
|
||||
stats.validated_at = td::Clocks::system();
|
||||
stats.valid = valid;
|
||||
if (valid) {
|
||||
stats.comment = (PSTRING() << "OK ts=" << now_);
|
||||
} else {
|
||||
stats.comment = std::move(error_message);
|
||||
}
|
||||
stats.actual_bytes = block_candidate.data.size();
|
||||
stats.actual_collated_data_bytes = block_candidate.collated_data.size();
|
||||
stats.total_time = perf_timer_.elapsed();
|
||||
stats.work_time = work_timer_.elapsed();
|
||||
stats.cpu_work_time = cpu_work_timer_.elapsed();
|
||||
LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s";
|
||||
LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s";
|
||||
td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time,
|
||||
cpu_work_time);
|
||||
LOG(WARNING) << "Validate query work time = " << stats.work_time << "s, cpu time = " << stats.cpu_work_time << "s";
|
||||
td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats));
|
||||
}
|
||||
|
||||
} // namespace validator
|
||||
|
|
|
|||
|
|
@ -407,7 +407,7 @@ class ValidateQuery : public td::actor::Actor {
|
|||
|
||||
td::Timer work_timer_{true};
|
||||
td::ThreadCpuTimer cpu_work_timer_{true};
|
||||
void record_stats();
|
||||
void record_stats(bool valid, std::string error_message = "");
|
||||
};
|
||||
|
||||
} // namespace validator
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@
|
|||
#include "auto/tl/lite_api.h"
|
||||
#include "impl/out-msg-queue-proof.hpp"
|
||||
|
||||
#include <ton/ton-tl.hpp>
|
||||
|
||||
namespace ton {
|
||||
|
||||
namespace validator {
|
||||
|
|
@ -54,7 +56,14 @@ struct AsyncSerializerState {
|
|||
};
|
||||
|
||||
struct CollationStats {
|
||||
BlockIdExt block_id;
|
||||
td::Bits256 collated_data_hash = td::Bits256::zero();
|
||||
CatchainSeqno cc_seqno = 0;
|
||||
double collated_at = -1.0;
|
||||
td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0;
|
||||
int attempt = 0;
|
||||
td::Bits256 collator_node_id = td::Bits256::zero();
|
||||
td::Bits256 validator_id = td::Bits256::zero();
|
||||
td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0;
|
||||
int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0;
|
||||
std::string limits_log;
|
||||
|
|
@ -62,15 +71,51 @@ struct CollationStats {
|
|||
td::uint32 ext_msgs_filtered = 0;
|
||||
td::uint32 ext_msgs_accepted = 0;
|
||||
td::uint32 ext_msgs_rejected = 0;
|
||||
double work_time = 0.0, cpu_work_time = 0.0;
|
||||
td::uint32 serialized_size = 0, serialized_size_no_collated_data = 0;
|
||||
double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0;
|
||||
std::string time_stats;
|
||||
|
||||
tl_object_ptr<ton_api::validatorSession_collationStats> tl() const {
|
||||
return create_tl_object<ton_api::validatorSession_collationStats>(
|
||||
actual_bytes, actual_collated_data_bytes, estimated_bytes, gas, lt_delta, estimated_collated_data_bytes,
|
||||
cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, limits_log, ext_msgs_total, ext_msgs_filtered,
|
||||
ext_msgs_accepted, ext_msgs_rejected, work_time, cpu_work_time, serialized_size,
|
||||
serialized_size_no_collated_data);
|
||||
tl_object_ptr<ton_api::validatorStats_collatedBlock> tl() const {
|
||||
int flags = (collator_node_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::COLLATOR_NODE_ID_MASK) |
|
||||
(validator_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::VALIDATOR_ID_MASK);
|
||||
return create_tl_object<ton_api::validatorStats_collatedBlock>(
|
||||
flags, create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes,
|
||||
actual_collated_data_bytes, attempt, collator_node_id, validator_id, total_time, work_time, cpu_work_time,
|
||||
time_stats,
|
||||
create_tl_object<ton_api::validatorStats_blockLimitsStatus>(estimated_bytes, gas, lt_delta,
|
||||
estimated_collated_data_bytes, cat_bytes, cat_gas,
|
||||
cat_lt_delta, cat_collated_data_bytes, limits_log),
|
||||
create_tl_object<ton_api::validatorStats_extMsgsStats>(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted,
|
||||
ext_msgs_rejected));
|
||||
}
|
||||
};
|
||||
|
||||
struct ValidationStats {
|
||||
BlockIdExt block_id;
|
||||
td::Bits256 collated_data_hash = td::Bits256::zero();
|
||||
double validated_at = -1.0;
|
||||
bool valid = false;
|
||||
std::string comment;
|
||||
td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0;
|
||||
double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0;
|
||||
|
||||
tl_object_ptr<ton_api::validatorStats_validatedBlock> tl() const {
|
||||
return create_tl_object<ton_api::validatorStats_validatedBlock>(
|
||||
create_tl_block_id(block_id), collated_data_hash, validated_at, valid, comment, actual_bytes,
|
||||
actual_collated_data_bytes, total_time, work_time, cpu_work_time);
|
||||
}
|
||||
};
|
||||
|
||||
struct CollatorNodeResponseStats {
|
||||
td::Bits256 collator_node_id = td::Bits256::zero();
|
||||
td::Bits256 validator_id = td::Bits256::zero();
|
||||
double timestamp = -1.0;
|
||||
BlockIdExt block_id, original_block_id;
|
||||
td::Bits256 collated_data_hash = td::Bits256::zero();
|
||||
|
||||
tl_object_ptr<ton_api::validatorStats_collatorNodeResponse> tl() const {
|
||||
return create_tl_object<ton_api::validatorStats_collatorNodeResponse>(
|
||||
collator_node_id, validator_id, timestamp, create_tl_block_id(block_id), create_tl_block_id(original_block_id),
|
||||
collated_data_hash);;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -198,9 +243,12 @@ class ValidatorManager : public ValidatorManagerInterface {
|
|||
|
||||
virtual void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) = 0;
|
||||
|
||||
virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0;
|
||||
virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0;
|
||||
virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) = 0;
|
||||
virtual void log_validator_session_stats(validatorsession::ValidatorSessionStats stats) {
|
||||
}
|
||||
virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) {
|
||||
}
|
||||
virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) {
|
||||
}
|
||||
|
||||
virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise<ConstBlockHandle> promise) = 0;
|
||||
virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise<td::Ref<BlockData>> promise) = 0;
|
||||
|
|
@ -220,9 +268,11 @@ class ValidatorManager : public ValidatorManagerInterface {
|
|||
virtual void add_lite_query_stats(int lite_query_id) {
|
||||
}
|
||||
|
||||
virtual void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) {
|
||||
virtual void log_collate_query_stats(CollationStats stats) {
|
||||
}
|
||||
virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) {
|
||||
virtual void log_validate_query_stats(ValidationStats stats) {
|
||||
}
|
||||
virtual void log_collator_node_response_stats(CollatorNodeResponseStats stats) {
|
||||
}
|
||||
|
||||
virtual void add_persistent_state_description(td::Ref<PersistentStateDescription> desc) = 0;
|
||||
|
|
|
|||
|
|
@ -129,7 +129,8 @@ void ValidatorManagerImpl::sync_complete(td::Promise<td::Unit> promise) {
|
|||
Ed25519_PublicKey created_by{td::Bits256::zero()};
|
||||
td::as<td::uint32>(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8);
|
||||
run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref<CollatorOptions>{true},
|
||||
actor_id(this), td::Timestamp::in(10.0), std::move(P), td::CancellationToken{}, 0);
|
||||
actor_id(this), td::Timestamp::in(10.0), std::move(P), adnl::AdnlNodeIdShort::zero(),
|
||||
td::CancellationToken{}, 0);
|
||||
}
|
||||
|
||||
void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector<BlockIdExt> prev, BlockIdExt last,
|
||||
|
|
|
|||
|
|
@ -403,15 +403,6 @@ class ValidatorManagerImpl : public ValidatorManager {
|
|||
void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint64> promise) override {
|
||||
if (queue_size_counter_.empty()) {
|
||||
queue_size_counter_ = td::actor::create_actor<QueueSizeCounter>("queuesizecounter", td::Ref<MasterchainState>{},
|
||||
|
|
|
|||
|
|
@ -468,15 +468,6 @@ class ValidatorManagerImpl : public ValidatorManager {
|
|||
void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint64> promise) override {
|
||||
if (queue_size_counter_.empty()) {
|
||||
queue_size_counter_ = td::actor::create_actor<QueueSizeCounter>("queuesizecounter", td::Ref<MasterchainState>{},
|
||||
|
|
|
|||
|
|
@ -3107,61 +3107,20 @@ void ValidatorManagerImpl::wait_shard_client_state(BlockSeqno seqno, td::Timesta
|
|||
shard_client_waiters_[seqno].waiting_.emplace_back(timeout, 0, std::move(promise));
|
||||
}
|
||||
|
||||
void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id,
|
||||
validatorsession::ValidatorSessionStats stats) {
|
||||
void ValidatorManagerImpl::log_validator_session_stats(validatorsession::ValidatorSessionStats stats) {
|
||||
std::string fname = opts_->get_session_logs_file();
|
||||
if (fname.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<tl_object_ptr<ton_api::validatorSession_statsRound>> rounds;
|
||||
for (const auto &round : stats.rounds) {
|
||||
std::vector<tl_object_ptr<ton_api::validatorSession_statsProducer>> producers;
|
||||
for (const auto &producer : round.producers) {
|
||||
BlockIdExt cur_block_id{block_id.id, producer.root_hash, producer.file_hash};
|
||||
auto it = recorded_block_stats_.find(cur_block_id);
|
||||
tl_object_ptr<ton_api::validatorSession_collationStats> collation_stats;
|
||||
if (it != recorded_block_stats_.end() && it->second.collator_stats_) {
|
||||
auto &stats = it->second.collator_stats_.value();
|
||||
collation_stats = stats.tl();
|
||||
}
|
||||
std::string approvers, signers;
|
||||
for (bool x : producer.approvers) {
|
||||
approvers += (x ? '1' : '0');
|
||||
}
|
||||
for (bool x : producer.signers) {
|
||||
signers += (x ? '1' : '0');
|
||||
}
|
||||
producers.push_back(create_tl_object<ton_api::validatorSession_statsProducer>(
|
||||
producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.root_hash,
|
||||
producer.file_hash, producer.comment, producer.block_timestamp, producer.is_accepted, producer.is_ours,
|
||||
producer.got_submit_at, producer.collation_time, producer.collated_at, producer.collation_cached,
|
||||
it == recorded_block_stats_.end() ? -1.0 : it->second.collator_work_time_,
|
||||
it == recorded_block_stats_.end() ? -1.0 : it->second.collator_cpu_work_time_, std::move(collation_stats),
|
||||
producer.validation_time, producer.validated_at, producer.validation_cached,
|
||||
it == recorded_block_stats_.end() ? -1.0 : it->second.validator_work_time_,
|
||||
it == recorded_block_stats_.end() ? -1.0 : it->second.validator_cpu_work_time_, producer.gen_utime,
|
||||
producer.approved_weight, producer.approved_33pct_at, producer.approved_66pct_at, std::move(approvers),
|
||||
producer.signed_weight, producer.signed_33pct_at, producer.signed_66pct_at, std::move(signers),
|
||||
producer.serialize_time, producer.deserialize_time, producer.serialized_size));
|
||||
}
|
||||
rounds.push_back(create_tl_object<ton_api::validatorSession_statsRound>(round.timestamp, std::move(producers)));
|
||||
}
|
||||
|
||||
auto obj = create_tl_object<ton_api::validatorSession_stats>(
|
||||
stats.success, create_tl_block_id(block_id), stats.timestamp, stats.self.bits256_value(), stats.session_id,
|
||||
stats.cc_seqno, stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures,
|
||||
stats.signatures_weight, stats.approve_signatures, stats.approve_signatures_weight, stats.first_round,
|
||||
std::move(rounds));
|
||||
stats.fix_block_ids();
|
||||
auto obj = stats.tl();
|
||||
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
|
||||
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
|
||||
|
||||
std::ofstream file;
|
||||
file.open(fname, std::ios_base::app);
|
||||
file << s << "\n";
|
||||
file.close();
|
||||
|
||||
LOG(INFO) << "Writing validator session stats for " << block_id.id.to_str();
|
||||
LOG(INFO) << "Writing validator session stats for " << stats.block_id.to_str();
|
||||
}
|
||||
|
||||
void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) {
|
||||
|
|
@ -3169,22 +3128,13 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa
|
|||
if (fname.empty()) {
|
||||
return;
|
||||
}
|
||||
std::vector<tl_object_ptr<ton_api::validatorSession_newValidatorGroupStats_node>> nodes;
|
||||
for (const auto &node : stats.nodes) {
|
||||
nodes.push_back(
|
||||
create_tl_object<ton_api::validatorSession_newValidatorGroupStats_node>(node.id.bits256_value(), node.weight));
|
||||
}
|
||||
auto obj = create_tl_object<ton_api::validatorSession_newValidatorGroupStats>(
|
||||
stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.last_key_block_seqno,
|
||||
stats.timestamp, stats.self_idx, std::move(nodes));
|
||||
auto obj = stats.tl();
|
||||
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
|
||||
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
|
||||
|
||||
std::ofstream file;
|
||||
file.open(fname, std::ios_base::app);
|
||||
file << s << "\n";
|
||||
file.close();
|
||||
|
||||
LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str()
|
||||
<< " cc_seqno=" << stats.cc_seqno;
|
||||
}
|
||||
|
|
@ -3194,21 +3144,13 @@ void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndVa
|
|||
if (fname.empty()) {
|
||||
return;
|
||||
}
|
||||
std::vector<tl_object_ptr<ton_api::validatorSession_endValidatorGroupStats_node>> nodes;
|
||||
for (const auto &node : stats.nodes) {
|
||||
nodes.push_back(create_tl_object<ton_api::validatorSession_endValidatorGroupStats_node>(node.id.bits256_value(),
|
||||
node.catchain_blocks));
|
||||
}
|
||||
auto obj = create_tl_object<ton_api::validatorSession_endValidatorGroupStats>(stats.session_id, stats.timestamp,
|
||||
std::move(nodes));
|
||||
auto obj = stats.tl();
|
||||
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
|
||||
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
|
||||
|
||||
std::ofstream file;
|
||||
file.open(fname, std::ios_base::app);
|
||||
file << s << "\n";
|
||||
file.close();
|
||||
|
||||
LOG(INFO) << "Writing end validator group stats for " << stats.session_id;
|
||||
}
|
||||
|
||||
|
|
@ -3657,45 +3599,49 @@ td::actor::ActorOwn<ValidatorManagerInterface> ValidatorManagerFactory::create(
|
|||
rldp, overlays);
|
||||
}
|
||||
|
||||
void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, CollationStats stats) {
|
||||
auto &record = new_block_stats_record(block_id);
|
||||
record.collator_work_time_ = stats.work_time;
|
||||
record.collator_cpu_work_time_ = stats.cpu_work_time;
|
||||
record.collator_stats_ = std::move(stats);
|
||||
|
||||
void ValidatorManagerImpl::log_collate_query_stats(CollationStats stats) {
|
||||
std::string fname = opts_->get_session_logs_file();
|
||||
if (fname.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto obj = create_tl_object<ton_api::validatorSession_statsCollatedBlock>(td::Clocks::system(),
|
||||
create_tl_block_id(block_id), stats.tl());
|
||||
auto obj = stats.tl();
|
||||
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
|
||||
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
|
||||
|
||||
std::ofstream file;
|
||||
file.open(fname, std::ios_base::app);
|
||||
file << s << "\n";
|
||||
file.close();
|
||||
|
||||
LOG(DEBUG) << "Writing collation stats stats for " << block_id.id.to_str();
|
||||
LOG(DEBUG) << "Writing collation stats stats for " << stats.block_id.to_str();
|
||||
}
|
||||
|
||||
void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) {
|
||||
auto &record = new_block_stats_record(block_id);
|
||||
record.validator_work_time_ = work_time;
|
||||
record.validator_cpu_work_time_ = cpu_work_time;
|
||||
}
|
||||
|
||||
ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) {
|
||||
if (!recorded_block_stats_.count(block_id)) {
|
||||
recorded_block_stats_lru_.push(block_id);
|
||||
if (recorded_block_stats_lru_.size() > 4096) {
|
||||
recorded_block_stats_.erase(recorded_block_stats_lru_.front());
|
||||
recorded_block_stats_lru_.pop();
|
||||
}
|
||||
void ValidatorManagerImpl::log_validate_query_stats(ValidationStats stats) {
|
||||
std::string fname = opts_->get_session_logs_file();
|
||||
if (fname.empty()) {
|
||||
return;
|
||||
}
|
||||
return recorded_block_stats_[block_id];
|
||||
auto obj = stats.tl();
|
||||
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
|
||||
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
|
||||
std::ofstream file;
|
||||
file.open(fname, std::ios_base::app);
|
||||
file << s << "\n";
|
||||
file.close();
|
||||
LOG(DEBUG) << "Writing validation stats stats for " << stats.block_id.to_str();
|
||||
}
|
||||
|
||||
void ValidatorManagerImpl::log_collator_node_response_stats(CollatorNodeResponseStats stats) {
|
||||
std::string fname = opts_->get_session_logs_file();
|
||||
if (fname.empty()) {
|
||||
return;
|
||||
}
|
||||
auto obj = stats.tl();
|
||||
auto s = td::json_encode<std::string>(td::ToJson(*obj.get()), false);
|
||||
s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end());
|
||||
std::ofstream file;
|
||||
file.open(fname, std::ios_base::app);
|
||||
file << s << "\n";
|
||||
file.close();
|
||||
LOG(DEBUG) << "Writing collator node response stats stats for " << stats.block_id.to_str();
|
||||
}
|
||||
|
||||
size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) {
|
||||
|
|
|
|||
|
|
@ -630,7 +630,7 @@ class ValidatorManagerImpl : public ValidatorManager {
|
|||
|
||||
void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise<td::Unit> promise) override;
|
||||
|
||||
void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override;
|
||||
void log_validator_session_stats(validatorsession::ValidatorSessionStats stats) override;
|
||||
void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override;
|
||||
void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override;
|
||||
|
||||
|
|
@ -775,19 +775,9 @@ class ValidatorManagerImpl : public ValidatorManager {
|
|||
|
||||
td::actor::ActorOwn<CandidatesBuffer> candidates_buffer_;
|
||||
|
||||
struct RecordedBlockStats {
|
||||
double collator_work_time_ = -1.0;
|
||||
double collator_cpu_work_time_ = -1.0;
|
||||
td::optional<CollationStats> collator_stats_;
|
||||
double validator_work_time_ = -1.0;
|
||||
double validator_cpu_work_time_ = -1.0;
|
||||
};
|
||||
std::map<BlockIdExt, RecordedBlockStats> recorded_block_stats_;
|
||||
std::queue<BlockIdExt> recorded_block_stats_lru_;
|
||||
|
||||
void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) override;
|
||||
void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override;
|
||||
RecordedBlockStats &new_block_stats_record(BlockIdExt block_id);
|
||||
void log_collate_query_stats(CollationStats stats) override;
|
||||
void log_validate_query_stats(ValidationStats stats) override;
|
||||
void log_collator_node_response_stats(CollatorNodeResponseStats stats) override;
|
||||
|
||||
std::map<PublicKeyHash, td::actor::ActorOwn<ValidatorTelemetry>> validator_telemetry_;
|
||||
|
||||
|
|
|
|||
|
|
@ -36,9 +36,8 @@ static bool need_send_candidate_broadcast(const validatorsession::BlockSourceInf
|
|||
!is_masterchain;
|
||||
}
|
||||
|
||||
void ValidatorGroup::generate_block_candidate(
|
||||
validatorsession::BlockSourceInfo source_info,
|
||||
td::Promise<validatorsession::ValidatorSession::GeneratedCandidate> promise) {
|
||||
void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo source_info,
|
||||
td::Promise<GeneratedCandidate> promise) {
|
||||
td::uint32 round_id = source_info.priority.round;
|
||||
if (round_id > last_known_round_id_) {
|
||||
last_known_round_id_ = round_id;
|
||||
|
|
@ -49,20 +48,21 @@ void ValidatorGroup::generate_block_candidate(
|
|||
}
|
||||
if (cached_collated_block_) {
|
||||
if (cached_collated_block_->result) {
|
||||
promise.set_value({cached_collated_block_->result.value().clone(), true});
|
||||
auto res = cached_collated_block_->result.value().clone();
|
||||
res.is_cached = true;
|
||||
promise.set_value(std::move(res));
|
||||
} else {
|
||||
cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) {
|
||||
return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), true};
|
||||
cached_collated_block_->promises.push_back(promise.wrap([](GeneratedCandidate &&res) {
|
||||
res.is_cached = true;
|
||||
return std::move(res);
|
||||
}));
|
||||
}
|
||||
return;
|
||||
}
|
||||
cached_collated_block_ = std::make_shared<CachedCollatedBlock>();
|
||||
cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) {
|
||||
return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), false};
|
||||
}));
|
||||
td::Promise<BlockCandidate> P = [SelfId = actor_id(this), cache = cached_collated_block_,
|
||||
source_info](td::Result<BlockCandidate> R) {
|
||||
cached_collated_block_->promises.push_back(std::move(promise));
|
||||
td::Promise<GeneratedCandidate> P = [SelfId = actor_id(this), cache = cached_collated_block_,
|
||||
source_info](td::Result<GeneratedCandidate> R) {
|
||||
td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache),
|
||||
std::move(R));
|
||||
};
|
||||
|
|
@ -75,7 +75,7 @@ void ValidatorGroup::generate_block_candidate(
|
|||
|
||||
void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info,
|
||||
std::shared_ptr<CachedCollatedBlock> cache,
|
||||
td::Result<BlockCandidate> R) {
|
||||
td::Result<GeneratedCandidate> R) {
|
||||
if (R.is_error()) {
|
||||
for (auto &p : cache->promises) {
|
||||
p.set_error(R.error().clone());
|
||||
|
|
@ -84,12 +84,12 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo
|
|||
cached_collated_block_ = nullptr;
|
||||
}
|
||||
} else {
|
||||
auto candidate = R.move_as_ok();
|
||||
add_available_block_candidate(candidate.pubkey.as_bits256(), candidate.id, candidate.collated_file_hash);
|
||||
auto c = R.move_as_ok();
|
||||
add_available_block_candidate(c.candidate.pubkey.as_bits256(), c.candidate.id, c.candidate.collated_file_hash);
|
||||
if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) {
|
||||
send_block_candidate_broadcast(candidate.id, candidate.data.clone());
|
||||
send_block_candidate_broadcast(c.candidate.id, c.candidate.data.clone());
|
||||
}
|
||||
cache->result = std::move(candidate);
|
||||
cache->result = std::move(c);
|
||||
for (auto &p : cache->promises) {
|
||||
p.set_value(cache->result.value().clone());
|
||||
}
|
||||
|
|
@ -186,7 +186,8 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so
|
|||
}
|
||||
auto next_block_id = create_next_block_id(root_hash, file_hash);
|
||||
LOG(WARNING) << "Accepted block " << next_block_id;
|
||||
td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, std::move(stats));
|
||||
stats.block_id = next_block_id;
|
||||
td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(stats));
|
||||
auto block =
|
||||
block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref<BlockData>{};
|
||||
|
||||
|
|
@ -293,7 +294,7 @@ std::unique_ptr<validatorsession::ValidatorSession::Callback> ValidatorGroup::ma
|
|||
std::move(candidate), std::move(P));
|
||||
}
|
||||
void on_generate_slot(validatorsession::BlockSourceInfo source_info,
|
||||
td::Promise<validatorsession::ValidatorSession::GeneratedCandidate> promise) override {
|
||||
td::Promise<GeneratedCandidate> promise) override {
|
||||
td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, std::move(source_info),
|
||||
std::move(promise));
|
||||
}
|
||||
|
|
@ -397,8 +398,8 @@ void ValidatorGroup::start(std::vector<BlockIdExt> prev, BlockIdExt min_masterch
|
|||
|
||||
for (auto &p : postponed_accept_) {
|
||||
auto next_block_id = create_next_block_id(p.root_hash, p.file_hash);
|
||||
td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id,
|
||||
std::move(p.stats));
|
||||
p.stats.block_id = next_block_id;
|
||||
td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(p.stats));
|
||||
|
||||
auto block =
|
||||
p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref<BlockData>{};
|
||||
|
|
@ -408,12 +409,11 @@ void ValidatorGroup::start(std::vector<BlockIdExt> prev, BlockIdExt min_masterch
|
|||
}
|
||||
postponed_accept_.clear();
|
||||
|
||||
validatorsession::NewValidatorGroupStats stats;
|
||||
stats.session_id = session_id_;
|
||||
stats.shard = shard_;
|
||||
stats.cc_seqno = validator_set_->get_catchain_seqno();
|
||||
stats.last_key_block_seqno = last_key_block_seqno_;
|
||||
stats.timestamp = td::Clocks::system();
|
||||
validatorsession::NewValidatorGroupStats stats{.session_id = session_id_,
|
||||
.shard = shard_,
|
||||
.cc_seqno = validator_set_->get_catchain_seqno(),
|
||||
.last_key_block_seqno = last_key_block_seqno_,
|
||||
.started_at = td::Clocks::system()};
|
||||
td::uint32 idx = 0;
|
||||
for (const auto &node : validator_set_->export_vector()) {
|
||||
PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id();
|
||||
|
|
@ -441,7 +441,8 @@ void ValidatorGroup::destroy() {
|
|||
return;
|
||||
}
|
||||
stats.cc_seqno = cc_seqno;
|
||||
td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id,
|
||||
stats.block_id = block_id;
|
||||
td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats,
|
||||
std::move(stats));
|
||||
});
|
||||
td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats,
|
||||
|
|
|
|||
|
|
@ -35,8 +35,7 @@ class ValidatorManager;
|
|||
|
||||
class ValidatorGroup : public td::actor::Actor {
|
||||
public:
|
||||
void generate_block_candidate(validatorsession::BlockSourceInfo source_info,
|
||||
td::Promise<validatorsession::ValidatorSession::GeneratedCandidate> promise);
|
||||
void generate_block_candidate(validatorsession::BlockSourceInfo source_info, td::Promise<GeneratedCandidate> promise);
|
||||
void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block,
|
||||
td::Promise<std::pair<UnixTime, bool>> promise);
|
||||
void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash,
|
||||
|
|
@ -146,14 +145,14 @@ class ValidatorGroup : public td::actor::Actor {
|
|||
bool monitoring_shard_ = true;
|
||||
|
||||
struct CachedCollatedBlock {
|
||||
td::optional<BlockCandidate> result;
|
||||
std::vector<td::Promise<BlockCandidate>> promises;
|
||||
td::optional<GeneratedCandidate> result;
|
||||
std::vector<td::Promise<GeneratedCandidate>> promises;
|
||||
};
|
||||
std::shared_ptr<CachedCollatedBlock> cached_collated_block_;
|
||||
td::CancellationTokenSource cancellation_token_source_;
|
||||
|
||||
void generated_block_candidate(validatorsession::BlockSourceInfo source_info,
|
||||
std::shared_ptr<CachedCollatedBlock> cache, td::Result<BlockCandidate> R);
|
||||
std::shared_ptr<CachedCollatedBlock> cache, td::Result<GeneratedCandidate> R);
|
||||
|
||||
using CacheKey = std::tuple<td::Bits256, BlockIdExt, FileHash, FileHash>;
|
||||
std::map<CacheKey, UnixTime> approved_candidates_cache_;
|
||||
|
|
@ -165,8 +164,7 @@ class ValidatorGroup : public td::actor::Actor {
|
|||
}
|
||||
|
||||
void get_validator_group_info_for_litequery_cont(
|
||||
td::uint32 expected_round,
|
||||
std::vector<tl_object_ptr<lite_api::liteServer_nonfinal_candidateInfo>> candidates,
|
||||
td::uint32 expected_round, std::vector<tl_object_ptr<lite_api::liteServer_nonfinal_candidateInfo>> candidates,
|
||||
td::Promise<tl_object_ptr<lite_api::liteServer_nonfinal_validatorGroupInfo>> promise);
|
||||
|
||||
std::set<std::tuple<td::Bits256, BlockIdExt, FileHash>> available_block_candidates_; // source, id, collated hash
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue