From 28699654fb35ad4f11204d0578832ba030850ed3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 21 Feb 2024 18:38:02 +0300 Subject: [PATCH] New private block overlays --- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 92172 -> 92852 bytes validator-engine/validator-engine.cpp | 12 + validator/CMakeLists.txt | 2 + validator/full-node-private-overlay-v2.cpp | 330 +++++++++++++++++++++ validator/full-node-private-overlay-v2.hpp | 109 +++++++ validator/full-node.cpp | 53 +++- validator/full-node.h | 2 + validator/full-node.hpp | 9 + 9 files changed, 505 insertions(+), 14 deletions(-) create mode 100644 validator/full-node-private-overlay-v2.cpp create mode 100644 validator/full-node-private-overlay-v2.hpp diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index e911eacf..aa761ad2 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -398,6 +398,7 @@ tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId; +tonNode.privateBlockOverlayIdV2 zero_state_file_hash:int256 workchain:int shard:long nodes:(vector int256) senders:(vector int256) = tonNode.PrivateBlockOverlayIdV2; tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool = tonNode.KeyBlocks; @@ -665,6 +666,7 @@ engine.validator.shardOverlayStats.neighbour id:string proto_verison:int capabil roundtrip:double unreliability:double has_state:string = engine.validator.shardOverlayStats.Neighbour; engine.validator.shardOverlayStats shard:string mode:string neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; +engine.validator.privateBlockOverlayV2Stats shard:string nodes:(vector int256) senders:(vector int256) created_at:int = engine.validator.PrivateBlockOverlayV2Stats; engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index afd8e0fb16b8b5e5cfd48d072aa4604134a85ceb..3afe3f33e8e089d6d7691e781d0441ed40885811 100644 GIT binary patch delta 304 zcmeCVz`ErsEAOM(`c@23Ah?m2Thid_(t|zPsd?#{d8vA3i8+}mi6!|(dI3e5Wr-!J zPC5C>+5TmzMLCI;VMf6vi6zCGBy}WIjuy;dlP}57^UF_3#irIXCCq5DfT`qWUBi49 z#%Y`9n#vclf^C|7aG9_oF0(-@ZXOfzVVPXGLVfaqMJkhHR`X2Wu$YylI5jV2^1?KE zkic{W1x8k`nu02Z-!bN~PV delta 52 zcmdmTm9^&rEAOM(`c@23z`v1~TXJ)Pq=>}k4Tc3Qj8ir%n#mV#c35#IYP$mqV@VnS Dn7a@P diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 5abf913a..c3d37424 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1937,6 +1937,10 @@ void ValidatorEngine::start_full_node() { td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, [](td::Unit) {}); } + for (auto &c : config_.collators) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, + ton::adnl::AdnlNodeIdShort(c.adnl_id)); + } } else { started_full_node(); } @@ -3647,6 +3651,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, ton::adnl::AdnlNodeIdShort(id), shard); } + if (!full_node_.empty()) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, + ton::adnl::AdnlNodeIdShort(id)); + } write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); @@ -3722,6 +3730,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollat td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::del_collator, ton::adnl::AdnlNodeIdShort(id), shard); } + if (!full_node_.empty()) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::del_collator_adnl_id, + ton::adnl::AdnlNodeIdShort(id)); + } write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index afeb7793..0cde874b 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -149,6 +149,8 @@ set(FULL_NODE_SOURCE full-node-master.cpp full-node-private-overlay.hpp full-node-private-overlay.cpp + full-node-private-overlay-v2.hpp + full-node-private-overlay-v2.cpp net/download-block.hpp net/download-block.cpp diff --git a/validator/full-node-private-overlay-v2.cpp b/validator/full-node-private-overlay-v2.cpp new file mode 100644 index 00000000..fedb8a78 --- /dev/null +++ b/validator/full-node-private-overlay-v2.cpp @@ -0,0 +1,330 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "full-node-private-overlay-v2.hpp" +#include "ton/ton-tl.hpp" +#include "common/delay.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" +#include "auto/tl/ton_api_json.h" + +namespace ton::validator::fullnode { + +void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) { + std::vector signatures; + for (auto &sig : query.signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } + + BlockIdExt block_id = create_block_id(query.id_); + BlockBroadcast B{block_id, + std::move(signatures), + static_cast(query.catchain_seqno_), + static_cast(query.validator_set_hash_), + std::move(query.data_), + std::move(query.proof_)}; + + auto P = td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); + } else { + LOG(INFO) << "dropped broadcast: " << R.move_as_error(); + } + } + }); + LOG(FULL_NODE_DEBUG) << "Got block broadcast in private overlay: " << B.block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B), + std::move(P)); +} + +void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash, ton_api::tonNode_newShardBlockBroadcast &query) { + BlockIdExt block_id = create_block_id(query.block_->block_); + LOG(FULL_NODE_DEBUG) << "Got block description broadcast in private overlay: " << block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, + query.block_->cc_seqno_, std::move(query.block_->data_)); +} + +void FullNodePrivateOverlayV2::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + return; + } + + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); +} + +void FullNodePrivateOverlayV2::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { + if (!inited_) { + return; + } + auto B = create_serialize_tl_object( + create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); + if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + } +} + +void FullNodePrivateOverlayV2::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + std::vector> sigs; + for (auto &sig : broadcast.signatures) { + sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone())); + } + auto B = create_serialize_tl_object( + create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs), + broadcast.proof.clone(), broadcast.data.clone()); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + +void FullNodePrivateOverlayV2::start_up() { + std::sort(nodes_.begin(), nodes_.end()); + nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); + + std::vector nodes, senders; + for (const adnl::AdnlNodeIdShort &id : nodes_) { + nodes.push_back(id.bits256_value()); + } + for (const adnl::AdnlNodeIdShort &id : senders_) { + senders.push_back(id.bits256_value()); + } + auto X = create_hash_tl_object( + zero_state_file_hash_, shard_.workchain, shard_.shard, std::move(nodes), std::move(senders)); + td::BufferSlice b{32}; + b.as_slice().copy_from(as_slice(X)); + overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; + overlay_id_ = overlay_id_full_.compute_short_id(); + + try_init(); +} + +void FullNodePrivateOverlayV2::try_init() { + // Sometimes adnl id is added to validator engine later (or not at all) + td::actor::send_closure( + adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok() && R.ok()) { + td::actor::send_closure(SelfId, &FullNodePrivateOverlayV2::init); + } else { + delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodePrivateOverlayV2::try_init); }, + td::Timestamp::in(30.0)); + } + }); +} + +void FullNodePrivateOverlayV2::init() { + LOG(FULL_NODE_INFO) << "Creating private block overlay for shard " << shard_.to_str() << ", adnl_id=" << local_id_ + << " : " << nodes_.size() << " nodes"; + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodePrivateOverlayV2::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void get_stats_extra(td::Promise promise) override { + td::actor::send_closure(node_, &FullNodePrivateOverlayV2::get_stats_extra, std::move(promise)); + } + Callback(td::actor::ActorId node) : node_(node) { + } + + private: + td::actor::ActorId node_; + }; + + std::map authorized_keys; + for (const adnl::AdnlNodeIdShort &sender : senders_) { + authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); + } + overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; + td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), + nodes_, std::make_unique(actor_id(this)), rules); + + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); + inited_ = true; +} + +void FullNodePrivateOverlayV2::tear_down() { + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + } +} + +void FullNodePrivateOverlayV2::get_stats_extra(td::Promise promise) { + auto res = create_tl_object(); + res->shard_ = shard_.to_str(); + for (const auto &x : nodes_) { + res->nodes_.push_back(x.bits256_value()); + } + for (const auto &x : senders_) { + res->senders_.push_back(x.bits256_value()); + } + res->created_at_ = created_at_; + promise.set_result(td::json_encode(td::ToJson(*res), true)); +} + +td::actor::ActorId FullNodePrivateBlockOverlays::choose_overlay(ShardIdFull shard) { + for (auto &p : id_to_overlays_) { + auto &overlays = p.second.overlays_; + ShardIdFull cur_shard = shard; + while (true) { + auto it = overlays.find(cur_shard); + if (it != overlays.end() && it->second.is_sender_) { + return it->second.overlay_.get(); + } + if (cur_shard.pfx_len() == 0) { + break; + } + cur_shard = shard_parent(cur_shard); + } + } + return {}; +} + +void FullNodePrivateBlockOverlays::update_overlays( + td::Ref state, std::set my_adnl_ids, const FileHash &zero_state_file_hash, + const td::actor::ActorId &keyring, const td::actor::ActorId &adnl, + const td::actor::ActorId &rldp, const td::actor::ActorId &rldp2, + const td::actor::ActorId &overlays, + const td::actor::ActorId &validator_manager) { + if (my_adnl_ids.empty()) { + id_to_overlays_.clear(); + return; + } + auto collators = state->get_collator_config(true); + auto all_validators = state->get_total_validator_set(0); + + struct OverlayInfo { + std::vector nodes, senders; + }; + std::map pverlay_infos; + + // Masterchain overlay: all validators + collators + OverlayInfo &mc_overlay = pverlay_infos[ShardIdFull(masterchainId)]; + for (const auto &x : all_validators->export_vector()) { + td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; + mc_overlay.nodes.emplace_back(addr); + mc_overlay.senders.emplace_back(addr); + } + for (const auto &x : collators.collator_nodes) { + mc_overlay.nodes.emplace_back(x.adnl_id); + } + + // Shard overlays: validators of the shard + collators of the shard + // See ValidatorManagerImpl::update_shards + std::set new_shards; + for (auto &v : state->get_shards()) { + ShardIdFull shard = v->shard(); + if (shard.is_masterchain()) { + continue; + } + if (v->before_split()) { + ShardIdFull l_shard{shard.workchain, shard_child(shard.shard, true)}; + ShardIdFull r_shard{shard.workchain, shard_child(shard.shard, false)}; + new_shards.insert(l_shard); + new_shards.insert(r_shard); + } else if (v->before_merge()) { + ShardIdFull p_shard{shard.workchain, shard_parent(shard.shard)}; + new_shards.insert(p_shard); + } else { + new_shards.insert(shard); + } + } + for (ShardIdFull shard : new_shards) { + auto val_set = state->get_validator_set(shard); + td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); + OverlayInfo &overlay = + pverlay_infos[shard_prefix_length(shard) <= min_split ? shard : shard_prefix(shard, min_split)]; + for (const auto &x : val_set->export_vector()) { + td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; + overlay.nodes.emplace_back(addr); + overlay.senders.emplace_back(addr); + } + } + for (auto &p : pverlay_infos) { + ShardIdFull shard = p.first; + OverlayInfo &overlay = p.second; + if (!shard.is_masterchain()) { + for (const auto &collator : collators.collator_nodes) { + if (shard_intersects(collator.shard, shard)) { + overlay.nodes.emplace_back(collator.adnl_id); + } + } + } + + std::sort(overlay.nodes.begin(), overlay.nodes.end()); + overlay.nodes.erase(std::unique(overlay.nodes.begin(), overlay.nodes.end()), overlay.nodes.end()); + std::sort(overlay.senders.begin(), overlay.senders.end()); + overlay.senders.erase(std::unique(overlay.senders.begin(), overlay.senders.end()), overlay.senders.end()); + } + + std::map old_private_block_overlays = std::move(id_to_overlays_); + id_to_overlays_.clear(); + + for (const auto &p : pverlay_infos) { + ShardIdFull shard = p.first; + const OverlayInfo &new_overlay_info = p.second; + for (adnl::AdnlNodeIdShort local_id : new_overlay_info.nodes) { + if (!my_adnl_ids.count(local_id)) { + continue; + } + Overlays::ShardOverlay &new_overlay = id_to_overlays_[local_id].overlays_[shard]; + Overlays::ShardOverlay &old_overlay = old_private_block_overlays[local_id].overlays_[shard]; + if (!old_overlay.overlay_.empty() && old_overlay.nodes_ == new_overlay_info.nodes && + old_overlay.senders_ == new_overlay_info.senders) { + new_overlay = std::move(old_overlay); + old_overlay = {}; + } else { + new_overlay.nodes_ = new_overlay_info.nodes; + new_overlay.senders_ = new_overlay_info.senders; + new_overlay.is_sender_ = std::binary_search(new_overlay.senders_.begin(), new_overlay.senders_.end(), local_id); + new_overlay.overlay_ = td::actor::create_actor( + "BlocksPrivateOverlay", local_id, shard, new_overlay.nodes_, new_overlay.senders_, zero_state_file_hash, + keyring, adnl, rldp, rldp2, overlays, validator_manager); + } + } + } + + // Delete old overlays, but not immediately + for (auto &p : old_private_block_overlays) { + for (auto &x : p.second.overlays_) { + if (x.second.overlay_.empty()) { + continue; + } + td::actor::ActorId id = x.second.overlay_.release(); + delay_action([id = std::move(id)]() { td::actor::send_closure(id, &FullNodePrivateOverlayV2::destroy); }, + td::Timestamp::in(60.0)); + } + } +} + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-private-overlay-v2.hpp b/validator/full-node-private-overlay-v2.hpp new file mode 100644 index 00000000..6fbcbbf7 --- /dev/null +++ b/validator/full-node-private-overlay-v2.hpp @@ -0,0 +1,109 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "full-node.h" + +namespace ton::validator::fullnode { + +class FullNodePrivateOverlayV2 : public td::actor::Actor { + public: + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + template + void process_broadcast(PublicKeyHash, T &) { + VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; + } + void receive_broadcast(PublicKeyHash src, td::BufferSlice query); + + void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); + void send_broadcast(BlockBroadcast broadcast); + + void start_up() override; + void tear_down() override; + + void destroy() { + stop(); + } + + FullNodePrivateOverlayV2(adnl::AdnlNodeIdShort local_id, ShardIdFull shard, std::vector nodes, + std::vector senders, FileHash zero_state_file_hash, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays, + td::actor::ActorId validator_manager) + : local_id_(local_id) + , shard_(shard) + , nodes_(std::move(nodes)) + , senders_(std::move(senders)) + , zero_state_file_hash_(zero_state_file_hash) + , keyring_(keyring) + , adnl_(adnl) + , rldp_(rldp) + , rldp2_(rldp2) + , overlays_(overlays) + , validator_manager_(validator_manager) { + } + + private: + adnl::AdnlNodeIdShort local_id_; + ShardIdFull shard_; + std::vector nodes_; + std::vector senders_; + FileHash zero_state_file_hash_; + + td::actor::ActorId keyring_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; + td::actor::ActorId overlays_; + td::actor::ActorId validator_manager_; + + bool inited_ = false; + overlay::OverlayIdFull overlay_id_full_; + overlay::OverlayIdShort overlay_id_; + UnixTime created_at_ = (UnixTime)td::Clocks::system(); + + void try_init(); + void init(); + void get_stats_extra(td::Promise promise); +}; + +class FullNodePrivateBlockOverlays { + public: + td::actor::ActorId choose_overlay(ShardIdFull shard); + void update_overlays(td::Ref state, std::set my_adnl_ids, + const FileHash& zero_state_file_hash, const td::actor::ActorId& keyring, + const td::actor::ActorId& adnl, const td::actor::ActorId& rldp, + const td::actor::ActorId& rldp2, + const td::actor::ActorId& overlays, + const td::actor::ActorId& validator_manager); + + private: + struct Overlays { + struct ShardOverlay { + td::actor::ActorOwn overlay_; + std::vector nodes_, senders_; + bool is_sender_ = false; + }; + std::map overlays_; + }; + + std::map id_to_overlays_; // local_id -> overlays +}; + +} // namespace ton::validator::fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index b8842a61..57323c4b 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -20,6 +20,7 @@ #include "ton/ton-io.hpp" #include "td/actor/MultiPromise.h" #include "full-node.h" +#include "common/delay.h" namespace ton { @@ -53,7 +54,7 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } } - create_private_block_overlay(key); + // create_private_block_overlay(key); promise.set_value(td::Unit()); } @@ -80,10 +81,20 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } } - private_block_overlays_.erase(key); + // private_block_overlays_.erase(key); promise.set_value(td::Unit()); } +void FullNodeImpl::add_collator_adnl_id(adnl::AdnlNodeIdShort id) { + ++local_collator_nodes_[id]; +} + +void FullNodeImpl::del_collator_adnl_id(adnl::AdnlNodeIdShort id) { + if (--local_collator_nodes_[id] == 0) { + local_collator_nodes_.erase(id); + } +} + void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) { auto it = shards_.find(shard_id); @@ -218,6 +229,19 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s ++it; } } + + std::set my_adnl_ids; + for (const auto &p : local_collator_nodes_) { + my_adnl_ids.insert(p.first); + } + for (auto key : local_keys_) { + auto it = current_validators_.find(key); + if (it != current_validators_.end()) { + my_adnl_ids.insert(it->second); + } + } + private_block_overlays_.update_overlays(state, std::move(my_adnl_ids), zero_state_file_hash_, keyring_, adnl_, rldp_, + rldp2_, overlays_, validator_manager_); } void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { @@ -262,9 +286,10 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; } - if (!private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_shard_block_info, - block_id, cc_seqno, data.clone()); + auto private_overlay = private_block_overlays_.choose_overlay(ShardIdFull(masterchainId)); + if (!private_overlay.empty()) { + td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_shard_block_info, block_id, cc_seqno, + data.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); } @@ -275,9 +300,9 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - if (!private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_broadcast, - broadcast.clone()); + auto private_overlay = private_block_overlays_.choose_overlay(broadcast.block_id.shard_full()); + if (!private_overlay.empty()) { + td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_broadcast, broadcast.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); } @@ -440,7 +465,7 @@ void FullNodeImpl::got_key_block_proof(td::Ref proof) { if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); - update_private_block_overlays(); + // update_private_block_overlays(); } if (keys == all_validators_) { @@ -483,7 +508,7 @@ void FullNodeImpl::got_zero_block_state(td::Ref state) { if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); - update_private_block_overlays(); + // update_private_block_overlays(); } if (keys == all_validators_) { @@ -613,7 +638,7 @@ void FullNodeImpl::start_up() { std::make_unique(actor_id(this)), std::move(started_promise_)); } -void FullNodeImpl::update_private_block_overlays() { +/* void FullNodeImpl::update_private_block_overlays() { private_block_overlays_.clear(); if (local_keys_.empty()) { return; @@ -624,7 +649,7 @@ void FullNodeImpl::update_private_block_overlays() { } void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { - /*CHECK(local_keys_.count(key)); + CHECK(local_keys_.count(key)); if (current_validators_.count(key)) { std::vector nodes; for (const auto &p : current_validators_) { @@ -633,8 +658,8 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { private_block_overlays_[key] = td::actor::create_actor( "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_); - }*/ -} + } +} */ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, diff --git a/validator/full-node.h b/validator/full-node.h index b10baeda..65e6fae0 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -63,6 +63,8 @@ class FullNode : public td::actor::Actor { virtual void add_permanent_key(PublicKeyHash key, td::Promise promise) = 0; virtual void del_permanent_key(PublicKeyHash key, td::Promise promise) = 0; + virtual void add_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; + virtual void del_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 5ed8101f..ff998f32 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -24,6 +24,7 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" #include "full-node-private-overlay.hpp" +#include "full-node-private-overlay-v2.hpp" #include #include @@ -42,6 +43,8 @@ class FullNodeImpl : public FullNode { void add_permanent_key(PublicKeyHash key, td::Promise promise) override; void del_permanent_key(PublicKeyHash key, td::Promise promise) override; + void add_collator_adnl_id(adnl::AdnlNodeIdShort id) override; + void del_collator_adnl_id(adnl::AdnlNodeIdShort id) override; void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, @@ -127,14 +130,20 @@ class FullNodeImpl : public FullNode { std::map current_validators_; std::set local_keys_; + std::map local_collator_nodes_; td::Promise started_promise_; FullNodeConfig config_; + // TODO: Decide what to do with old private overlays. Maybe use old or new depending on some flag in config. + /* std::map> private_block_overlays_; void update_private_block_overlays(); void create_private_block_overlay(PublicKeyHash key); + */ + + FullNodePrivateBlockOverlays private_block_overlays_; }; } // namespace fullnode