From 539d5dd2def40ed1096d3745f78f5f4b67a2fbf4 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 24 May 2024 09:58:07 +0300 Subject: [PATCH] Add candidates cache (#1000) * Broadcast shardchain block candidates in private overlays, generate proof links from candidates * Disable shardchain block broadcasts in private overlays * Send block candidate broadcasts to custom overlays, allow non-validators to receive candidates --------- Co-authored-by: SpyCheese --- create-hardfork/create-hardfork.cpp | 3 + test/test-ton-collator.cpp | 3 + tl/generate/scheme/ton_api.tl | 3 + tl/generate/scheme/ton_api.tlo | Bin 89388 -> 89652 bytes ton/ton-types.h | 4 + validator/db/files-async.hpp | 2 +- validator/downloaders/wait-block-data.cpp | 81 ++++++++++++++++++++- validator/downloaders/wait-block-data.hpp | 6 +- validator/downloaders/wait-block-state.cpp | 3 +- validator/full-node-private-overlay.cpp | 65 +++++++++++++++++ validator/full-node-private-overlay.hpp | 6 ++ validator/full-node-shard.cpp | 32 ++++++++ validator/full-node-shard.h | 2 + validator/full-node-shard.hpp | 4 + validator/full-node.cpp | 56 +++++++++++++- validator/full-node.h | 2 + validator/full-node.hpp | 7 ++ validator/impl/collator.cpp | 13 ++-- validator/impl/validate-query.cpp | 3 +- validator/interfaces/validator-manager.h | 3 +- validator/manager-disk.cpp | 3 +- validator/manager-disk.hpp | 5 +- validator/manager-hardfork.hpp | 6 +- validator/manager.cpp | 71 +++++++++++++++++- validator/manager.hpp | 16 +++- validator/validator.h | 3 + 26 files changed, 382 insertions(+), 20 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index a9e7403a..3110b1a8 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -246,6 +246,9 @@ class HardforkCreator : public td::actor::Actor { } void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override { } + void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + } void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index d36db5d2..60476cf8 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -347,6 +347,9 @@ class TestNode : public td::actor::Actor { } } } + void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + } void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index d0a063f7..5d1e5b50 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -396,6 +396,9 @@ tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int valida tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast; tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast; tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; +// signature may be empty, at least for now +tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast; tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index fb7f219765fea2ff1aa4dee29f67166b388d01f9..89948bb4acb26d8a5255b77af3a6710b2739f3b4 100644 GIT binary patch delta 245 zcmZ3pi*?H$)(r==CExjW)e4s6=lSKQr0O{p block) { + if (data_.not_null()) { + return; + } + data_ = std::move(block); + if (handle_->received()) { + finish_query(); + return; + } + if (!handle_->id().is_masterchain() && !handle_->inited_proof_link()) { + // This can happen if we get block from candidates cache. + // Proof link can be derived from the block (but not for masterchain block). + auto r_proof_link = generate_proof_link(handle_->id(), data_->root_cell()); + if (r_proof_link.is_error()) { + abort_query(r_proof_link.move_as_error_prefix("failed to create proof link for block: ")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::validate_block_proof_link, handle_->id(), + r_proof_link.move_as_ok(), + [id = handle_->id().id, SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockData::abort_query, + R.move_as_error_prefix("validate proof link error: ")); + return; + } + LOG(DEBUG) << "Created and validated proof link for " << id.to_str(); + td::actor::send_closure(SelfId, &WaitBlockData::checked_proof_link); + }); + return; + } + checked_proof_link(); +} + +void WaitBlockData::checked_proof_link() { CHECK(handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link()); if (!handle_->received()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -198,6 +238,41 @@ void WaitBlockData::got_static_file(td::BufferSlice data) { run_hardfork_accept_block_query(handle_->id(), data_, manager_, std::move(P)); } +td::Result WaitBlockData::generate_proof_link(BlockIdExt id, td::Ref block_root) { + // Creating proof link. Similar to accept-block.cpp + if (id.is_masterchain()) { + return td::Status::Error("cannot create proof link for masterchain block"); + } + auto usage_tree = std::make_shared(); + auto usage_cell = vm::UsageCell::create(block_root, usage_tree->root_ptr()); + + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + block::gen::BlockExtra::Record extra; + block::gen::ExtBlkRef::Record mcref{}; // _ ExtBlkRef = BlkMasterInfo; + ShardIdFull shard; + if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(blk.info, info) && !info.version && + block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && + block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) && + tlb::unpack_cell(std::move(blk.extra), extra) && block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) && + (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) { + return td::Status::Error("cannot unpack block header"); + } + vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update}; + + auto proof = vm::MerkleProof::generate(block_root, usage_tree.get()); + vm::CellBuilder cb; + td::Ref bs_cell; + if (!(cb.store_long_bool(0xc3, 8) // block_proof#c3 + && block::tlb::t_BlockIdExt.pack(cb, id) // proof_for:BlockIdExt + && cb.store_ref_bool(std::move(proof)) // proof:^Cell + && cb.store_bool_bool(false) // signatures:(Maybe ^BlockSignatures) + && cb.finalize_to(bs_cell))) { + return td::Status::Error("cannot serialize BlockProof"); + } + return std_boc_serialize(bs_cell, 0); +} + } // namespace validator } // namespace ton diff --git a/validator/downloaders/wait-block-data.hpp b/validator/downloaders/wait-block-data.hpp index 9a03b1cb..229b4bfc 100644 --- a/validator/downloaders/wait-block-data.hpp +++ b/validator/downloaders/wait-block-data.hpp @@ -57,11 +57,15 @@ class WaitBlockData : public td::actor::Actor { void set_is_hardfork(bool value); void start(); void got_block_data_from_db(td::Ref data); - void got_block_data_from_net(ReceivedBlock data); + void got_data_from_net(ReceivedBlock data); + void got_block_data_from_net(td::Ref block); + void checked_proof_link(); void failed_to_get_block_data_from_net(td::Status reason); void got_static_file(td::BufferSlice data); + static td::Result generate_proof_link(BlockIdExt id, td::Ref block_root); + private: BlockHandle handle_; diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 42f5c791..0ae82bea 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -192,7 +192,8 @@ void WaitBlockState::got_proof_link(td::BufferSlice data) { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); } else { LOG(INFO) << "received bad proof link: " << R.move_as_error(); - td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); + delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); }, + td::Timestamp::in(0.1)); } }); run_check_proof_link_query(handle_->id(), R.move_as_ok(), manager_, timeout_, std::move(P)); diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 74bb75c5..ffe5468d 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -17,6 +17,7 @@ #include "full-node-private-overlay.hpp" #include "ton/ton-tl.hpp" #include "common/delay.h" +#include "common/checksum.h" #include "full-node-serializer.hpp" namespace ton::validator::fullnode { @@ -49,6 +50,22 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api:: query.block_->cc_seqno_, std::move(query.block_->data_)); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcast &query) { + if (query.data_.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + BlockIdExt block_id = create_block_id(query.id_); + if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in private overlay from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, + query.validator_set_hash_, std::move(query.data_)); +} + void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { if (adnl::AdnlNodeIdShort{src} == local_id_) { return; @@ -77,6 +94,19 @@ void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, Cat } } +void FullNodePrivateBlockOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in private overlay: " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { if (!inited_) { return; @@ -199,6 +229,28 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNod std::move(query.message_->data_), it->second); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { + VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ + << "\" from unauthorized sender " << src; + return; + } + if (query.data_.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + BlockIdExt block_id = create_block_id(query.id_); + if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + // ignore cc_seqno and validator_hash for now + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in custom overlay \"" << name_ << "\" from " << src << ": " + << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, + query.validator_set_hash_, std::move(query.data_)); +} + void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { if (adnl::AdnlNodeIdShort{src} == local_id_) { return; @@ -241,6 +293,19 @@ void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) { local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } +void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in custom overlay \"" << name_ << "\": " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + void FullNodeCustomOverlay::start_up() { std::sort(nodes_.begin(), nodes_.end()); nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 196d6da6..e310824f 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -27,6 +27,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -34,6 +35,8 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast); void set_config(FullNodeConfig config) { @@ -98,6 +101,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -106,6 +110,8 @@ class FullNodeCustomOverlay : public td::actor::Actor { void send_external_message(td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); void set_config(FullNodeConfig config) { config_ = std::move(config); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 6e265cbc..9b2b321e 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -17,12 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "auto/tl/ton_api.h" +#include "checksum.h" #include "overlays.h" #include "td/utils/SharedSlice.h" #include "full-node-shard.hpp" #include "full-node-shard-queries.hpp" #include "full-node-serializer.hpp" +#include "td/utils/buffer.h" #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" @@ -646,6 +648,22 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne query.block_->cc_seqno_, std::move(query.block_->data_)); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + if (query.data_.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + BlockIdExt block_id = create_block_id(query.id_); + if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + // ignore cc_seqno and validator_hash for now + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, + query.validator_set_hash_, std::move(query.data_)); +} + void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { process_block_broadcast(src, query); } @@ -738,6 +756,20 @@ void FullNodeShardImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno } } +void FullNodeShardImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) { + if (!client_.empty()) { + UNREACHABLE(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate: " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_, + overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { if (!client_.empty()) { UNREACHABLE(); diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 2ed13d66..e89031fe 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -41,6 +41,8 @@ class FullNodeShard : public td::actor::Actor { virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast) = 0; virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 1fdbce1c..08f48f86 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -18,6 +18,7 @@ */ #pragma once +#include "auto/tl/ton_api.h" #include "full-node-shard.h" #include "td/actor/PromiseFuture.h" #include "td/utils/port/Poll.h" @@ -152,12 +153,15 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; void send_broadcast(BlockBroadcast broadcast) override; void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 8d0f26bc..ff8f1eca 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -243,6 +243,24 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); } +void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); + auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; + return; + } + if (!private_block_overlays_.empty()) { + td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, + block_id, cc_seqno, validator_set_hash, data.clone()); + } + if (broadcast_block_candidates_in_public_overlay_) { + td::actor::send_closure(shard, &FullNodeShard::send_block_candidate, block_id, cc_seqno, validator_set_hash, + std::move(data)); + } +} + void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) { send_block_broadcast_to_custom_overlays(broadcast); if (custom_overlays_only) { @@ -253,7 +271,7 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - if (!private_block_overlays_.empty()) { + if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, broadcast.clone()); } @@ -420,6 +438,14 @@ void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { }); } +void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); + // ignore cc_seqno and validator_hash for now + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate, block_id, + std::move(data)); +} + void FullNodeImpl::start_up() { add_shard(ShardIdFull{masterchainId}); if (local_id_.is_zero()) { @@ -452,6 +478,11 @@ void FullNodeImpl::start_up() { void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_shard_block_info, block_id, cc_seqno, std::move(data)); } + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + td::actor::send_closure(id_, &FullNodeImpl::send_block_candidate, block_id, cc_seqno, validator_set_hash, + std::move(data)); + } void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), custom_overlays_only); } @@ -587,6 +618,29 @@ void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& } } +void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt &block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, + const td::BufferSlice &data) { + // Same cache of sent broadcasts as in send_block_broadcast_to_custom_overlays + if (!custom_overlays_sent_broadcasts_.insert(block_id).second) { + return; + } + custom_overlays_sent_broadcasts_lru_.push(block_id); + if (custom_overlays_sent_broadcasts_lru_.size() > 256) { + custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); + custom_overlays_sent_broadcasts_lru_.pop(); + } + for (auto &private_overlay : custom_overlays_) { + for (auto &actor : private_overlay.second.actors_) { + auto local_id = actor.first; + if (private_overlay.second.params_.block_senders_.count(local_id)) { + td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_block_candidate, block_id, cc_seqno, + validator_set_hash, data.clone()); + } + } + } +} + FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/full-node.h b/validator/full-node.h index ac260dd7..c3719f67 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -87,6 +87,8 @@ class FullNode : public td::actor::Actor { virtual void del_custom_overlay(std::string name, td::Promise promise) = 0; virtual void process_block_broadcast(BlockBroadcast broadcast) = 0; + virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) = 0; static constexpr td::uint32 max_block_size() { return 4 << 20; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 946a1705..9b78b3e0 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -66,6 +66,8 @@ class FullNodeImpl : public FullNode { void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, @@ -84,6 +86,8 @@ class FullNodeImpl : public FullNode { void new_key_block(BlockHandle handle); void process_block_broadcast(BlockBroadcast broadcast) override; + void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; void start_up() override; @@ -124,6 +128,7 @@ class FullNodeImpl : public FullNode { std::map> private_block_overlays_; bool private_block_overlays_enable_compression_ = false; + bool broadcast_block_candidates_in_public_overlay_ = false; struct CustomOverlayInfo { CustomOverlayParams params_; @@ -138,6 +143,8 @@ class FullNodeImpl : public FullNode { void create_private_block_overlay(PublicKeyHash key); void update_custom_overlay(CustomOverlayInfo& overlay); void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); + void send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt& block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, const td::BufferSlice& data); }; } // namespace fullnode diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fd4ddd34..ba38ba42 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -5012,12 +5012,13 @@ bool Collator::create_block_candidate() { } // 4. save block candidate LOG(INFO) << "saving new BlockCandidate"; - td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, - block_candidate->clone(), [self = get_self()](td::Result saved) -> void { - LOG(DEBUG) << "got answer to set_block_candidate"; - td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, - std::move(saved)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::set_block_candidate, block_candidate->id, block_candidate->clone(), + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), + [self = get_self()](td::Result saved) -> void { + LOG(DEBUG) << "got answer to set_block_candidate"; + td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, std::move(saved)); + }); // 5. communicate about bad and delayed external messages if (!bad_ext_msgs_.empty() || !delay_ext_msgs_.empty()) { LOG(INFO) << "sending complete_external_messages() to Manager"; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8b2723c3..88bc6163 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -6330,7 +6330,8 @@ bool ValidateQuery::save_candidate() { } }); - td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), std::move(P)); + td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); return true; } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index d435b756..0e9fab73 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -93,7 +93,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) = 0; + virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) = 0; virtual void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 42e44081..17b793c7 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -775,7 +775,8 @@ void ValidatorManagerImpl::set_next_block(BlockIdExt block_id, BlockIdExt next, get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) { +void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) { td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 5cf93884..12bd68ef 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -130,6 +130,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); @@ -177,7 +179,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override; + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index fb5237d6..3a124f82 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -152,6 +152,9 @@ class ValidatorManagerImpl : public ValidatorManager { void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { UNREACHABLE(); } + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + UNREACHABLE(); + } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); @@ -215,7 +218,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override { + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override { promise.set_value(td::Unit()); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3d544003..f91f04a1 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -17,6 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "manager.hpp" +#include "checksum.h" +#include "td/utils/buffer.h" #include "validator-group.hpp" #include "adnl/utils.hpp" #include "downloaders/wait-block-state.hpp" @@ -464,6 +466,17 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc actor_id(this), td::Timestamp::in(2.0), std::move(P)); } +void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSlice data) { + if (!last_masterchain_block_handle_) { + VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; + return; + } + if (!started_) { + return; + } + add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)}); +} + void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { if (desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); @@ -495,6 +508,36 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refsecond.actor_, &WaitBlockData::got_block_data_from_net, r_block.move_as_ok()); + } + } + } + { + auto it = wait_state_.find(id); + if (it != wait_state_.end()) { + // Proof link is not ready at this point, but this will force WaitBlockState to redo send_get_proof_link_request + td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); + } + } + } + if (cached_block_candidates_lru_.size() > max_cached_candidates()) { + CHECK(cached_block_candidates_.erase(cached_block_candidates_lru_.front())); + cached_block_candidates_lru_.pop_front(); + } +} + void ValidatorManagerImpl::add_ext_server_id(adnl::AdnlNodeIdShort id) { class Cb : public adnl::Adnl::Callback { private: @@ -1189,11 +1232,16 @@ void ValidatorManagerImpl::set_next_block(BlockIdExt block_id, BlockIdExt next, get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) { +void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) { if (!candidates_buffer_.empty()) { td::actor::send_closure(candidates_buffer_, &CandidatesBuffer::add_new_candidate, id, PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.collated_file_hash); } + if (!id.is_masterchain()) { + add_cached_block_candidate(ReceivedBlock{id, candidate.data.clone()}); + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, candidate.data.clone()); + } td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } @@ -1450,6 +1498,13 @@ void ValidatorManagerImpl::get_last_liteserver_state_block( void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) { + { + auto it = cached_block_candidates_.find(id); + if (it != cached_block_candidates_.end()) { + LOG(DEBUG) << "send_get_block_request: got result from candidates cache for " << id.to_str(); + return promise.set_value(it->second.clone()); + } + } callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1472,6 +1527,20 @@ void ValidatorManagerImpl::send_get_block_proof_request(BlockIdExt block_id, td: void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) { + if (!block_id.is_masterchain()) { + auto it = cached_block_candidates_.find(block_id); + if (it != cached_block_candidates_.end()) { + // Proof link can be created from the cached block candidate + LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached caniddate for " + << block_id.to_str(); + TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(it->second.data), + "failed to create proof link: "); + TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(it->second.id, block_root), + "failed to create proof link: "); + promise.set_result(std::move(proof_link)); + return; + } + } callback_->download_block_proof_link(block_id, priority, td::Timestamp::in(10.0), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 65636a94..be7f18db 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -18,10 +18,14 @@ */ #pragma once +#include "common/refcnt.hpp" #include "interfaces/validator-manager.h" #include "interfaces/db.h" #include "td/actor/PromiseFuture.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/buffer.h" #include "td/utils/port/Poll.h" +#include "td/utils/port/StdStreams.h" #include "validator-group.hpp" #include "shard-client.hpp" #include "manager-init.h" @@ -220,6 +224,10 @@ class ValidatorManagerImpl : public ValidatorManager { }; // DATA FOR COLLATOR std::map> shard_blocks_; + + std::map cached_block_candidates_; + std::list cached_block_candidates_lru_; + struct ExtMessages { std::map, std::unique_ptr>> ext_messages_; std::map, std::map>> @@ -365,6 +373,7 @@ class ValidatorManagerImpl : public ValidatorManager { void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override; void add_ext_server_id(adnl::AdnlNodeIdShort id) override; void add_ext_server_port(td::uint16 port) override; @@ -409,7 +418,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override; + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -503,6 +513,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_shard_block_description(td::Ref desc); + void add_cached_block_candidate(ReceivedBlock block); void register_block_handle(BlockHandle handle); @@ -664,6 +675,9 @@ class ValidatorManagerImpl : public ValidatorManager { double max_mempool_num() const { return opts_->max_mempool_num(); } + size_t max_cached_candidates() const { + return 128; + } private: std::map> shard_client_waiters_; diff --git a/validator/validator.h b/validator/validator.h index 245acbd4..4e3390cc 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -142,6 +142,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only = false) = 0; virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; @@ -210,6 +212,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void check_external_message(td::BufferSlice data, td::Promise> promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) = 0; virtual void add_ext_server_id(adnl::AdnlNodeIdShort id) = 0; virtual void add_ext_server_port(td::uint16 port) = 0;