external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-02-15 04:32:21 +00:00
Code Issues Releases Wiki Activity

Improve CollatorNode

Browse source 
* Keep track of validator groups
* Pre-generate shard blocks
This commit is contained in:
SpyCheese 2024-06-25 14:06:15 +03:00
parent 3695bf0797
commit 90d2edf535
12 changed files with 410 additions and 174 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
tl/generate/scheme/ton_api.tl
View file

@ -837,7 +837,7 @@ validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:lo
self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats;
---functions---
collatorNode.generateBlock workchain:int shard:long min_mc_id:tonNode.blockIdExt prev_blocks:(vector tonNode.blockIdExt)
collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt)
creator:int256 = collatorNode.GenerateBlockResult;
---types---

BIN
tl/generate/scheme/ton_api.tlo
View file

Binary file not shown.

4
validator-engine/validator-engine.cpp
View file

@ -575,8 +575,8 @@ td::Result<bool> Config::config_add_control_process(ton::PublicKeyHash key, td::
}
td::Result<bool> Config::config_add_shard(ton::ShardIdFull shard) {
if (!shard.is_valid_ext()) {
return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str());
if (!shard.is_valid_ext() || shard.is_masterchain()) {
return td::Status::Error(PSTRING() << "invalid shard to collate " << shard.to_str());
}
if (std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard) != shards_to_monitor.end()) {
return false;

380
validator/collator-node.cpp
View file

@ -21,6 +21,7 @@
#include "block-db.h"
#include "td/utils/lz4.h"
#include "checksum.h"
#include "impl/shard.hpp"
#include "validator-session/candidate-serializer.h"
namespace ton::validator {
@ -33,7 +34,7 @@ CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::actor::ActorId<Va
void CollatorNode::start_up() {
class Cb : public adnl::Adnl::Callback {
public:
Cb(td::actor::ActorId<CollatorNode> id) : id_(std::move(id)) {
explicit Cb(td::actor::ActorId<CollatorNode> id) : id_(std::move(id)) {
}
void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override {
}
@ -57,54 +58,195 @@ void CollatorNode::tear_down() {
}
void CollatorNode::add_shard(ShardIdFull shard) {
if (std::find(shards_.begin(), shards_.end(), shard) != shards_.end()) {
CHECK(shard.is_valid_ext() && !shard.is_masterchain());
if (std::find(collating_shards_.begin(), collating_shards_.end(), shard) != collating_shards_.end()) {
return;
}
LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str();
shards_.push_back(shard);
collating_shards_.push_back(shard);
}
void CollatorNode::del_shard(ShardIdFull shard) {
auto it = std::find(shards_.begin(), shards_.end(), shard);
if (it != shards_.end()) {
shards_.erase(it);
auto it = std::find(collating_shards_.begin(), collating_shards_.end(), shard);
if (it != collating_shards_.end()) {
collating_shards_.erase(it);
}
}
void CollatorNode::new_masterchain_block_notification(td::Ref<MasterchainState> state) {
last_masterchain_block_ = state->get_block_id();
last_top_blocks_.clear();
last_top_blocks_[ShardIdFull{masterchainId, shardIdAll}] = last_masterchain_block_;
for (const auto& desc : state->get_shards()) {
last_top_blocks_[desc->shard()] = desc->top_block_id();
}
if (validators_.empty() || state->is_key_state()) {
validators_.clear();
last_masterchain_state_ = state;
if (validator_adnl_ids_.empty() || state->is_key_state()) {
validator_adnl_ids_.clear();
for (int next : {-1, 0, 1}) {
td::Ref<ValidatorSet> vals = state->get_total_validator_set(next);
if (vals.not_null()) {
for (const ValidatorDescr& descr : vals->export_vector()) {
if (descr.addr.is_zero()) {
validators_.insert(
validator_adnl_ids_.insert(
adnl::AdnlNodeIdShort(PublicKey(pubkeys::Ed25519{descr.key.as_bits256()}).compute_short_id()));
} else {
validators_.insert(adnl::AdnlNodeIdShort(descr.addr));
validator_adnl_ids_.insert(adnl::AdnlNodeIdShort(descr.addr));
}
}
}
}
}
// Remove old cache entries
auto it = cache_.begin();
while (it != cache_.end()) {
auto prev_block_id = std::get<2>(it->first)[0];
auto top_block = get_shard_top_block(prev_block_id.shard_full());
if (top_block && top_block.value().seqno() > prev_block_id.seqno()) {
it = cache_.erase(it);
std::map<ShardIdFull, std::vector<BlockIdExt>> new_shards;
for (auto& v : state->get_shards()) {
auto shard = v->shard();
if (v->before_split()) {
CHECK(!v->before_merge());
new_shards.emplace(shard_child(shard, true), std::vector{v->top_block_id()});
new_shards.emplace(shard_child(shard, false), std::vector{v->top_block_id()});
} else if (v->before_merge()) {
ShardIdFull p_shard = shard_parent(shard);
auto it = new_shards.find(p_shard);
if (it == new_shards.end()) {
new_shards.emplace(p_shard, std::vector<BlockIdExt>(2));
}
bool left = shard_child(p_shard.shard, true) == shard.shard;
new_shards[p_shard][left ? 0 : 1] = v->top_block_id();
} else {
++it;
new_shards.emplace(shard, std::vector{v->top_block_id()});
}
}
for (auto& [shard, prev] : new_shards) {
CatchainSeqno cc_seqno = state->get_validator_set(shard)->get_catchain_seqno();
auto it = validator_groups_.emplace(shard, ValidatorGroupInfo{});
ValidatorGroupInfo& info = it.first->second;
if (it.second || info.cc_seqno != cc_seqno) {
info.cleanup();
info.cc_seqno = cc_seqno;
}
}
for (auto it = validator_groups_.begin(); it != validator_groups_.end();) {
if (new_shards.count(it->first)) {
++it;
} else {
it->second.cleanup();
it = validator_groups_.erase(it);
}
}
for (auto& [shard, prev] : new_shards) {
ValidatorGroupInfo& info = validator_groups_[shard];
update_validator_group_info(shard, std::move(prev), info.cc_seqno);
auto it = future_validator_groups_.find({shard, info.cc_seqno});
if (it != future_validator_groups_.end()) {
for (auto& new_prev : it->second.pending_blocks) {
update_validator_group_info(shard, std::move(new_prev), info.cc_seqno);
}
for (auto& promise : it->second.promises) {
promise.set_value(td::Unit());
}
future_validator_groups_.erase(it);
}
}
for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end(); ++it) {
if (get_future_validator_group(it->first.first, it->first.second).is_ok()) {
++it;
} else {
auto& future_group = it->second;
for (auto& promise : future_group.promises) {
promise.set_error(td::Status::Error("validator group is outdated"));
}
it = future_validator_groups_.erase(it);
}
}
}
void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector<BlockIdExt> prev,
CatchainSeqno cc_seqno) {
if (!can_collate_shard(shard)) {
return;
}
CHECK(prev.size() == 1 || prev.size() == 2);
BlockSeqno next_block_seqno = prev[0].seqno() + 1;
if (prev.size() == 2) {
next_block_seqno = std::max(next_block_seqno, prev[1].seqno() + 1);
}
auto it = validator_groups_.find(shard);
if (it != validator_groups_.end()) {
ValidatorGroupInfo& info = it->second;
if (info.cc_seqno == cc_seqno) { // block from currently known validator group
if (info.next_block_seqno < next_block_seqno) {
LOG(DEBUG) << "updated validator group info: shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno
<< ", next_block_seqno=" << next_block_seqno;
info.next_block_seqno = next_block_seqno;
info.prev = std::move(prev);
for (auto cache_it = info.cache.begin(); cache_it != info.cache.end();) {
auto& [cached_prev, cache_entry] = *cache_it;
if (cache_entry->block_seqno < info.next_block_seqno) {
cache_entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << cache_entry->block_seqno
<< " is too small, expected " << info.next_block_seqno));
cache_it = info.cache.erase(cache_it);
continue;
}
if (cache_entry->block_seqno == info.next_block_seqno && cached_prev != info.prev) {
cache_entry->cancel(td::Status::Error("invalid prev blocks"));
cache_it = info.cache.erase(cache_it);
continue;
}
++cache_it;
}
if (do_pregen_) {
generate_block(shard, cc_seqno, info.prev, td::Timestamp::in(10.0), [](td::Result<BlockCandidate>) {});
}
}
return;
}
}
auto future_validator_group = get_future_validator_group(shard, cc_seqno);
if (future_validator_group.is_ok()) {
// future validator group, remember for later
future_validator_group.ok()->pending_blocks.push_back(std::move(prev));
}
}
td::Result<CollatorNode::FutureValidatorGroup*> CollatorNode::get_future_validator_group(ShardIdFull shard,
CatchainSeqno cc_seqno) {
auto it = validator_groups_.find(shard);
if (it == validator_groups_.end() && shard.pfx_len() != 0) {
it = validator_groups_.find(shard_parent(shard));
}
if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) {
it = validator_groups_.find(shard_child(shard, true));
}
if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) {
it = validator_groups_.find(shard_child(shard, false));
}
if (it == validator_groups_.end()) {
return td::Status::Error("no such shard");
}
if (cc_seqno < it->second.cc_seqno) { // past validator group
return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is outdated (current is" << it->second.cc_seqno
<< ")");
}
if (cc_seqno - it->second.cc_seqno > 1) { // future validator group, cc_seqno too big
return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is too big (currently known is"
<< it->second.cc_seqno << ")");
}
// future validator group
return &future_validator_groups_[{shard, cc_seqno}];
}
void CollatorNode::ValidatorGroupInfo::cleanup() {
prev.clear();
next_block_seqno = 0;
for (auto& [_, cache_entry] : cache) {
cache_entry->cancel(td::Status::Error("validator group is outdated"));
}
cache.clear();
}
void CollatorNode::CacheEntry::cancel(td::Status reason) {
for (auto& promise : promises) {
promise.set_error(reason.clone());
}
promises.clear();
cancellation_token_source.cancel();
}
static td::BufferSlice serialize_error(td::Status error) {
@ -136,111 +278,121 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data
td::Promise<td::BufferSlice> promise) {
td::Promise<BlockCandidate> new_promise = [promise = std::move(promise), src](td::Result<BlockCandidate> R) mutable {
if (R.is_error()) {
LOG(WARNING) << "Query from " << src << ", error: " << R.error();
promise.set_result(serialize_error(R.move_as_error()));
LOG(INFO) << "adnl query from " << src << ", error: " << R.error();
if (R.error().code() == ErrorCode::timeout) {
promise.set_error(R.move_as_error());
} else {
promise.set_result(serialize_error(R.move_as_error()));
}
} else {
LOG(INFO) << "Query from " << src << ", success";
LOG(INFO) << "adnl query from " << src << ", success";
promise.set_result(create_serialize_tl_object<ton_api::collatorNode_generateBlockSuccess>(
serialize_candidate(R.move_as_ok(), true)));
}
};
if (!last_masterchain_block_.is_valid()) {
new_promise.set_error(td::Status::Error("not ready"));
return;
}
if (!validators_.count(src)) {
if (!validator_adnl_ids_.count(src)) {
new_promise.set_error(td::Status::Error("src is not a validator"));
}
TRY_RESULT_PROMISE(new_promise, f, fetch_tl_object<ton_api::collatorNode_generateBlock>(std::move(data), true));
ShardIdFull shard(f->workchain_, f->shard_);
BlockIdExt min_mc_id = create_block_id(f->min_mc_id_);
LOG(INFO) << "Got query from " << src << ": shard=" << shard.to_str() << ", min_mc_seqno=" << min_mc_id.seqno();
ShardIdFull shard = create_shard_id(f->shard_);
CatchainSeqno cc_seqno = f->cc_seqno_;
std::vector<BlockIdExt> prev_blocks;
for (const auto& b : f->prev_blocks_) {
prev_blocks.push_back(create_block_id(b));
}
Ed25519_PublicKey creator(f->creator_);
new_promise = [new_promise = std::move(new_promise), creator](td::Result<BlockCandidate> R) mutable {
TRY_RESULT_PROMISE(new_promise, block, std::move(R));
new_promise.set_value(change_creator(std::move(block), creator));
};
if (!shard.is_valid_ext()) {
new_promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()));
return;
}
if (!can_collate_shard(shard)) {
new_promise.set_error(td::Status::Error(PSTRING() << "this node doesn't collate shard " << shard.to_str()));
if (prev_blocks.size() != 1 && prev_blocks.size() != 2) {
new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << prev_blocks.size()));
return;
}
if (f->prev_blocks_.size() != 1 && f->prev_blocks_.size() != 2) {
new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << f->prev_blocks_.size()));
return;
}
if (!min_mc_id.is_masterchain_ext()) {
new_promise.set_error(td::Status::Error("min_mc_id is not form masterchain"));
return;
}
std::vector<BlockIdExt> prev_blocks;
for (const auto& b : f->prev_blocks_) {
auto id = create_block_id(b);
if (!id.is_valid_full()) {
new_promise.set_error(td::Status::Error("invalid prev_block"));
return;
}
auto top_block = get_shard_top_block(id.shard_full());
if (top_block && top_block.value().seqno() > id.seqno()) {
new_promise.set_error(td::Status::Error("cannot collate block: already exists in blockchain"));
return;
}
prev_blocks.push_back(id);
}
Ed25519_PublicKey creator(f->creator_);
if (!shard.is_masterchain()) {
// Collation of masterchain cannot be cached because changing "created_by" in masterchain is hard
// It does not really matter because validators can collate masterchain themselves
new_promise = [promise = std::move(new_promise), creator](td::Result<BlockCandidate> R) mutable {
if (R.is_error()) {
promise.set_error(R.move_as_error());
} else {
promise.set_result(change_creator(R.move_as_ok(), creator));
}
};
auto cache_key = std::make_tuple(min_mc_id.seqno(), shard, prev_blocks);
auto cache_entry = cache_[cache_key];
if (cache_entry == nullptr) {
cache_entry = cache_[cache_key] = std::make_shared<CacheEntry>();
}
if (cache_entry->result) {
LOG(INFO) << "Using cached result";
new_promise.set_result(cache_entry->result.value().clone());
return;
}
cache_entry->promises.push_back(std::move(new_promise));
if (cache_entry->started) {
LOG(INFO) << "Collating of this block is already in progress, waiting";
return;
}
cache_entry->started = true;
new_promise = [SelfId = actor_id(this), cache_entry](td::Result<BlockCandidate> R) mutable {
td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R));
};
}
auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks),
promise = std::move(new_promise)](td::Result<td::Ref<ShardState>> R) mutable {
if (R.is_error()) {
promise.set_error(R.move_as_error_prefix("failed to get masterchain state: "));
} else {
td::Ref<MasterchainState> state(R.move_as_ok());
if (state.is_null()) {
promise.set_error(R.move_as_error_prefix("failed to get masterchain state: "));
return;
}
td::actor::send_closure(SelfId, &CollatorNode::receive_query_cont, shard, std::move(state),
std::move(prev_blocks), creator, std::move(promise));
}
});
td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, min_mc_id, 1, td::Timestamp::in(5.0),
std::move(P));
LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno;
generate_block(shard, cc_seqno, std::move(prev_blocks), td::Timestamp::in(10.0), std::move(new_promise));
}
void CollatorNode::receive_query_cont(ShardIdFull shard, td::Ref<MasterchainState> min_mc_state,
std::vector<BlockIdExt> prev_blocks, Ed25519_PublicKey creator,
td::Promise<BlockCandidate> promise) {
run_collate_query(shard, min_mc_state->get_block_id(), std::move(prev_blocks), creator,
min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(promise),
CollateMode::skip_store_candidate);
void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector<BlockIdExt> prev_blocks,
td::Timestamp timeout, td::Promise<BlockCandidate> promise) {
if (last_masterchain_state_.is_null()) {
promise.set_error(td::Status::Error(ErrorCode::notready, "not ready"));
return;
}
if (!can_collate_shard(shard)) {
promise.set_error(td::Status::Error(PSTRING() << "this node can't collate shard " << shard.to_str()));
return;
}
auto it = validator_groups_.find(shard);
if (it == validator_groups_.end() || it->second.cc_seqno != cc_seqno) {
TRY_RESULT_PROMISE(promise, future_validator_group, get_future_validator_group(shard, cc_seqno));
future_validator_group->promises.push_back([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks),
promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
promise.set_error(R.move_as_error());
return;
}
if (timeout.is_in_past()) {
promise.set_error(td::Status::Error(ErrorCode::timeout));
return;
}
td::actor::send_closure(SelfId, &CollatorNode::generate_block, shard, cc_seqno, std::move(prev_blocks), timeout,
std::move(promise));
});
return;
}
ValidatorGroupInfo& validator_group_info = it->second;
BlockSeqno block_seqno = prev_blocks.at(0).seqno() + 1;
if (prev_blocks.size() == 2) {
block_seqno = std::max(block_seqno, prev_blocks.at(1).seqno() + 1);
}
if (validator_group_info.next_block_seqno > block_seqno) {
promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too small, expected "
<< validator_group_info.next_block_seqno));
return;
}
if (validator_group_info.next_block_seqno == block_seqno && validator_group_info.prev != prev_blocks) {
promise.set_error(td::Status::Error("invalid prev_blocks"));
return;
}
auto cache_entry = validator_group_info.cache[prev_blocks];
if (cache_entry == nullptr) {
cache_entry = validator_group_info.cache[prev_blocks] = std::make_shared<CacheEntry>();
}
if (cache_entry->result) {
LOG(INFO) << "generate block query"
<< ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno
<< ": using cached result";
promise.set_result(cache_entry->result.value().clone());
return;
}
cache_entry->promises.push_back(std::move(promise));
if (cache_entry->started) {
LOG(INFO) << "generate block query"
<< ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno
<< ": collation in progress, waiting";
return;
}
LOG(INFO) << "generate block query"
<< ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno
<< ": starting collation";
cache_entry->started = true;
cache_entry->block_seqno = block_seqno;
run_collate_query(
shard, last_masterchain_state_->get_block_id(), std::move(prev_blocks), Ed25519_PublicKey{td::Bits256::zero()},
last_masterchain_state_->get_validator_set(shard), manager_, timeout,
[=, SelfId = actor_id(this), timer = td::Timer{}](td::Result<BlockCandidate> R) {
LOG(INFO) << "generate block result"
<< ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno
<< ", time=" << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string());
td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R));
},
cache_entry->cancellation_token_source.get_cancellation_token(), CollateMode::skip_store_candidate);
}
void CollatorNode::process_result(std::shared_ptr<CacheEntry> cache_entry, td::Result<BlockCandidate> R) {
@ -259,7 +411,7 @@ void CollatorNode::process_result(std::shared_ptr<CacheEntry> cache_entry, td::R
}
bool CollatorNode::can_collate_shard(ShardIdFull shard) const {
return std::any_of(shards_.begin(), shards_.end(),
return std::any_of(collating_shards_.begin(), collating_shards_.end(),
[&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); });
}

48
validator/collator-node.hpp
View file

@ -34,12 +34,10 @@ class CollatorNode : public td::actor::Actor {
void del_shard(ShardIdFull shard);
void new_masterchain_block_notification(td::Ref<MasterchainState> state);
void update_validator_group_info(ShardIdFull shard, std::vector<BlockIdExt> prev, CatchainSeqno cc_seqno);
private:
void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise<td::BufferSlice> promise);
void receive_query_cont(ShardIdFull shard, td::Ref<MasterchainState> min_mc_state,
std::vector<BlockIdExt> prev_blocks, Ed25519_PublicKey creator,
td::Promise<BlockCandidate> promise);
bool can_collate_shard(ShardIdFull shard) const;
@ -47,33 +45,39 @@ class CollatorNode : public td::actor::Actor {
td::actor::ActorId<ValidatorManager> manager_;
td::actor::ActorId<adnl::Adnl> adnl_;
td::actor::ActorId<rldp::Rldp> rldp_;
std::vector<ShardIdFull> shards_;
std::set<adnl::AdnlNodeIdShort> validators_;
BlockIdExt last_masterchain_block_{};
std::map<ShardIdFull, BlockIdExt> last_top_blocks_;
std::vector<ShardIdFull> collating_shards_;
std::set<adnl::AdnlNodeIdShort> validator_adnl_ids_;
struct CacheEntry {
bool started = false;
BlockSeqno block_seqno = 0;
td::optional<BlockCandidate> result;
td::CancellationTokenSource cancellation_token_source;
std::vector<td::Promise<BlockCandidate>> promises;
void cancel(td::Status reason);
};
std::map<std::tuple<BlockSeqno, ShardIdFull, std::vector<BlockIdExt>>, std::shared_ptr<CacheEntry>> cache_;
struct ValidatorGroupInfo {
CatchainSeqno cc_seqno{0};
std::vector<BlockIdExt> prev;
BlockSeqno next_block_seqno{0};
std::map<std::vector<BlockIdExt>, std::shared_ptr<CacheEntry>> cache;
td::optional<BlockIdExt> get_shard_top_block(ShardIdFull shard) const {
auto it = last_top_blocks_.lower_bound(shard);
if (it != last_top_blocks_.end() && shard_intersects(it->first, shard)) {
return it->second;
}
if (it != last_top_blocks_.begin()) {
--it;
if (shard_intersects(it->first, shard)) {
return it->second;
}
}
return {};
}
void cleanup();
};
struct FutureValidatorGroup {
std::vector<std::vector<BlockIdExt>> pending_blocks;
std::vector<td::Promise<td::Unit>> promises;
};
std::map<ShardIdFull, ValidatorGroupInfo> validator_groups_;
std::map<std::pair<ShardIdFull, CatchainSeqno>, FutureValidatorGroup> future_validator_groups_;
td::Ref<MasterchainState> last_masterchain_state_;
td::Result<FutureValidatorGroup*> get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno);
void generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector<BlockIdExt> prev_blocks,
td::Timestamp timeout, td::Promise<BlockCandidate> promise);
void process_result(std::shared_ptr<CacheEntry> cache_entry, td::Result<BlockCandidate> R);
public:

3
validator/fabric.h
View file

@ -88,7 +88,8 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id,
void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector<BlockIdExt> prev,
Ed25519_PublicKey creator, td::Ref<ValidatorSet> validator_set,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
td::Promise<BlockCandidate> promise, unsigned mode = 0);
td::Promise<BlockCandidate> promise, td::CancellationToken cancellation_token = {},
unsigned mode = 0);
void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector<BlockIdExt> prev,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
td::Promise<BlockCandidate> promise);

9
validator/impl/collator-impl.h
View file

@ -87,9 +87,9 @@ class Collator final : public td::actor::Actor {
static constexpr bool shard_splitting_enabled = true;
public:
Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id,
std::vector<BlockIdExt> prev, Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout, td::Promise<BlockCandidate> promise,
Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev,
Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId<ValidatorManager> manager,
td::Timestamp timeout, td::Promise<BlockCandidate> promise, td::CancellationToken cancellation_token,
unsigned mode);
~Collator() override = default;
bool is_busy() const {
@ -343,6 +343,9 @@ class Collator final : public td::actor::Actor {
void return_block_candidate(td::Result<td::Unit> saved);
bool update_last_proc_int_msg(const std::pair<ton::LogicalTime, ton::Bits256>& new_lt_hash);
td::CancellationToken cancellation_token_;
bool check_cancelled();
public:
static td::uint32 get_skip_externals_queue_size();
};

37
validator/impl/collator.cpp
View file

@ -77,7 +77,7 @@ static inline bool dbg(int c) {
Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id,
std::vector<BlockIdExt> prev, td::Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
td::Promise<BlockCandidate> promise, unsigned mode)
td::Promise<BlockCandidate> promise, td::CancellationToken cancellation_token, unsigned mode)
: shard_(shard)
, is_hardfork_(is_hardfork)
, min_mc_block_id{min_masterchain_block_id}
@ -92,9 +92,11 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha
, medium_timeout_(td::Timestamp::at(timeout.at() - 1.5))
, main_promise(std::move(promise))
, mode_(mode)
, perf_timer_("collate", 0.1, [manager](double duration) {
send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration);
}) {
, perf_timer_("collate", 0.1,
[manager](double duration) {
send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration);
})
, cancellation_token_(std::move(cancellation_token)) {
}
/**
@ -382,6 +384,9 @@ bool Collator::fatal_error(std::string err_msg, int err_code) {
*/
void Collator::check_pending() {
// LOG(DEBUG) << "pending = " << pending;
if (!check_cancelled()) {
return;
}
if (!pending) {
step = 2;
try {
@ -2423,6 +2428,9 @@ bool Collator::out_msg_queue_cleanup() {
LOG(WARNING) << "cleaning up outbound queue takes too long, ending";
break;
}
if (!check_cancelled()) {
return false;
}
if (i == queue_parts.size()) {
i = 0;
}
@ -3516,6 +3524,9 @@ bool Collator::process_inbound_internal_messages() {
LOG(WARNING) << "soft timeout reached, stop processing inbound internal messages";
break;
}
if (!check_cancelled()) {
return false;
}
LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex()
<< ") from neighbor #" << kv->source;
if (verbosity > 2) {
@ -3565,6 +3576,9 @@ bool Collator::process_inbound_external_messages() {
LOG(WARNING) << "medium timeout reached, stop processing inbound external messages";
break;
}
if (!check_cancelled()) {
return false;
}
auto ext_msg = ext_msg_struct.cell;
ton::Bits256 hash{ext_msg->get_hash().bits()};
int r = process_external_message(std::move(ext_msg));
@ -3819,6 +3833,9 @@ bool Collator::process_new_messages(bool enqueue_only) {
LOG(INFO) << "BLOCK FULL, enqueue all remaining new messages";
enqueue_only = true;
}
if (!check_cancelled()) {
return false;
}
LOG(DEBUG) << "have message with lt=" << msg.lt;
int res = process_one_new_message(std::move(msg), enqueue_only);
if (res < 0) {
@ -5360,6 +5377,18 @@ void Collator::after_get_external_messages(td::Result<std::vector<std::pair<Ref<
check_pending();
}
/**
* Checks if collation was cancelled via cancellation token
*
* @returns false if the collation was cancelled, true otherwise
*/
bool Collator::check_cancelled() {
if (cancellation_token_) {
return fatal_error(td::Status::Error(ErrorCode::cancelled, "cancelled"));
}
return true;
}
td::uint32 Collator::get_skip_externals_queue_size() {
return SKIP_EXTERNALS_QUEUE_SIZE;
}

7
validator/impl/fabric.cpp
View file

@ -215,7 +215,7 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id,
void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector<BlockIdExt> prev,
Ed25519_PublicKey creator, td::Ref<ValidatorSet> validator_set,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
td::Promise<BlockCandidate> promise, unsigned mode) {
td::Promise<BlockCandidate> promise, td::CancellationToken cancellation_token, unsigned mode) {
BlockSeqno seqno = 0;
for (auto& p : prev) {
if (p.seqno() > seqno) {
@ -224,7 +224,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc
}
td::actor::create_actor<Collator>(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false,
min_masterchain_block_id, std::move(prev), std::move(validator_set), creator,
std::move(manager), timeout, std::move(promise), mode)
std::move(manager), timeout, std::move(promise), std::move(cancellation_token),
mode)
.release();
}
@ -240,7 +241,7 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b
td::actor::create_actor<Collator>(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true,
min_masterchain_block_id, std::move(prev), td::Ref<ValidatorSet>{},
Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise),
0)
td::CancellationToken{}, 0)
.release();
}

88
validator/manager.cpp
View file

@ -218,12 +218,28 @@ void ValidatorManagerImpl::prevalidate_block(BlockBroadcast broadcast, td::Promi
promise.set_error(td::Status::Error("not monitoring shard"));
return;
}
promise = [SelfId = actor_id(this), promise = std::move(promise), block_id = broadcast.block_id,
cc_seqno = broadcast.catchain_seqno](td::Result<td::Unit> R) mutable {
if (R.is_ok()) {
td::actor::send_closure(SelfId, &ValidatorManagerImpl::validated_block_broadcast, block_id, cc_seqno);
}
promise.set_result(std::move(R));
};
td::actor::create_actor<ValidateBroadcast>("broadcast", std::move(broadcast), last_masterchain_block_handle_,
last_masterchain_state_, last_known_key_block_handle_, actor_id(this),
td::Timestamp::in(2.0), std::move(promise))
.release();
}
void ValidatorManagerImpl::validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno) {
for (auto &[_, collator_node] : collator_nodes_) {
if (collator_node.can_collate_shard(block_id.shard_full())) {
td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, block_id.shard_full(),
std::vector{block_id}, cc_seqno);
}
}
}
void ValidatorManagerImpl::sync_complete(td::Promise<td::Unit> promise) {
started_ = true;
@ -471,7 +487,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) {
}
void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) {
if (!is_validator()) {
if (!is_validator() && !is_shard_collator(block_id.shard_full())) {
return;
}
if (!last_masterchain_block_handle_) {
@ -513,29 +529,36 @@ void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSl
}
void ValidatorManagerImpl::add_shard_block_description(td::Ref<ShardTopBlockDescription> desc) {
if (desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) {
auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()});
if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second.latest_desc->block_id().id.seqno) {
VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast";
return;
}
shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}].latest_desc = desc;
VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id();
if (need_monitor(desc->block_id().shard_full())) {
auto P = td::PromiseCreator::lambda([](td::Result<td::Ref<ShardState>> R) {
if (R.is_error()) {
auto S = R.move_as_error();
if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) {
VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S;
} else {
VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S;
}
if (!desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) {
return;
}
auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()});
if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second.latest_desc->block_id().id.seqno) {
VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast";
return;
}
shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}].latest_desc = desc;
VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id();
if (need_monitor(desc->block_id().shard_full())) {
auto P = td::PromiseCreator::lambda([](td::Result<td::Ref<ShardState>> R) {
if (R.is_error()) {
auto S = R.move_as_error();
if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) {
VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S;
} else {
VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S;
}
});
wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), std::move(P));
}
if (validating_masterchain()) {
preload_msg_queue_to_masterchain(desc);
}
});
wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), std::move(P));
}
if (validating_masterchain()) {
preload_msg_queue_to_masterchain(desc);
}
for (auto& [_, collator_node] : collator_nodes_) {
if (collator_node.can_collate_shard(desc->shard())) {
td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, desc->shard(),
std::vector{desc->block_id()}, desc->catchain_seqno());
}
}
}
@ -2863,6 +2886,21 @@ PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref<Val
return PublicKeyHash::zero();
}
bool ValidatorManagerImpl::is_shard_collator(ShardIdFull shard) {
for (auto &[_, collator_node] : collator_nodes_) {
if (collator_node.can_collate_shard(shard)) {
return true;
}
}
return false;
}
bool ValidatorManagerImpl::Collator::can_collate_shard(ShardIdFull shard) const {
return std::any_of(shards.begin(), shards.end(),
[&](const ShardIdFull &our_shard) { return shard_intersects(shard, our_shard); });
}
void ValidatorManagerImpl::got_next_key_blocks(std::vector<BlockIdExt> r) {
if (r.size() == 0) {
delay_action([SelfId = actor_id(
@ -3364,6 +3402,10 @@ void ValidatorManagerImpl::get_validator_sessions_info(
}
void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) {
if (shard.is_masterchain() || !shard.is_valid_ext()) {
LOG(WARNING) << "cannot collate shard " << shard.to_str();
return;
}
auto it = collator_nodes_.find(id);
if (it == collator_nodes_.end()) {
it = collator_nodes_.emplace(id, Collator()).first;

4
validator/manager.hpp
View file

@ -372,6 +372,7 @@ class ValidatorManagerImpl : public ValidatorManager {
td::Promise<td::Unit> promise) override;
void validate_block(ReceivedBlock block, td::Promise<BlockHandle> promise) override;
void prevalidate_block(BlockBroadcast broadcast, td::Promise<td::Unit> promise) override;
void validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno);
//void create_validate_block(BlockId block, td::BufferSlice data, td::Promise<Block> promise) = 0;
void sync_complete(td::Promise<td::Unit> promise) override;
@ -562,6 +563,7 @@ class ValidatorManagerImpl : public ValidatorManager {
bool is_validator();
bool validating_masterchain();
PublicKeyHash get_validator(ShardIdFull shard, td::Ref<ValidatorSet> val_set);
bool is_shard_collator(ShardIdFull shard);
ValidatorManagerImpl(td::Ref<ValidatorManagerOptions> opts, std::string db_root,
td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
@ -763,6 +765,8 @@ class ValidatorManagerImpl : public ValidatorManager {
struct Collator {
td::actor::ActorOwn<CollatorNode> actor;
std::set<ShardIdFull> shards;
bool can_collate_shard(ShardIdFull shard) const;
};
std::map<adnl::AdnlNodeIdShort, Collator> collator_nodes_;

2
validator/validator-group.cpp
View file

@ -546,7 +546,7 @@ void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, t
prev_blocks.push_back(create_tl_block_id(p));
}
td::BufferSlice query = create_serialize_tl_object<ton_api::collatorNode_generateBlock>(
shard_.workchain, shard_.shard, create_tl_block_id(min_masterchain_block_id_), std::move(prev_blocks),
create_tl_shard_id(shard_), validator_set_->get_catchain_seqno(), std::move(prev_blocks),
local_id_full_.ed25519_value().raw());
auto P = td::PromiseCreator::lambda(