Improve block broadcasts processing; add special overlay for blocks for validators (#885)

* Improve block broadcast processing

* ValidatorManagerImpl::written_handle
* Retry sending broadcasts in ValidatorGroup
* Fix setting channel_ready in AdnlPeerPair

* Add special overlay for validators for block broadcasting (#842)

* Private overlay for broadcasting blocks

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>

(cherry picked from commit a52045bd91)

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>

adnl/adnl-peer-table.hpp

@@ -77,6 +77,10 @@ class AdnlPeerTableImpl : public AdnlPeerTable {
                         td::actor::ActorId<AdnlChannel> channel) override;
   void unregister_channel(AdnlChannelIdShort id) override;
 
+  void check_id_exists(AdnlNodeIdShort id, td::Promise<bool> promise) override {
+    promise.set_value(local_ids_.count(id));
+  }
+
   void write_new_addr_list_to_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, AdnlDbItem node,
                                  td::Promise<td::Unit> promise) override;
   void get_addr_list_from_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id,

adnl/adnl-peer.cpp

@@ -212,7 +212,9 @@ void AdnlPeerPairImpl::receive_packet_from_channel(AdnlChannelIdShort id, AdnlPa
     VLOG(ADNL_NOTICE) << this << ": dropping IN message: outdated channel id" << id;
     return;
   }
+  if (channel_inited_) {
     channel_ready_ = true;
+  }
   receive_packet_checked(std::move(packet));
 }
 

adnl/adnl.h

@@ -97,6 +97,8 @@ class Adnl : public AdnlSenderInterface {
   virtual void add_id_ex(AdnlNodeIdFull id, AdnlAddressList addr_list, td::uint8 cat, td::uint32 mode) = 0;
   virtual void del_id(AdnlNodeIdShort id, td::Promise<td::Unit> promise) = 0;
 
+  virtual void check_id_exists(AdnlNodeIdShort id, td::Promise<bool> promise) = 0;
+
   // subscribe to (some) messages(+queries) to this local id
   virtual void subscribe(AdnlNodeIdShort dst, std::string prefix, std::unique_ptr<Callback> callback) = 0;
   virtual void unsubscribe(AdnlNodeIdShort dst, std::string prefix) = 0;

tl/generate/scheme/ton_api.tl

@@ -394,6 +394,8 @@ tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast;
 
 tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId;
 
+tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId;
+
 tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool = tonNode.KeyBlocks;
 
 ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId;

ton/ton-types.h

@@ -353,6 +353,14 @@ struct BlockBroadcast {
   td::uint32 validator_set_hash;
   td::BufferSlice data;
   td::BufferSlice proof;
+
+  BlockBroadcast clone() const {
+    std::vector<BlockSignature> new_signatures;
+    for (const BlockSignature& s : signatures) {
+      new_signatures.emplace_back(s.node, s.signature.clone());
+    }
+    return {block_id, std::move(new_signatures), catchain_seqno, validator_set_hash, data.clone(), proof.clone()};
+  }
 };
 
 struct Ed25519_PrivateKey {

validator/CMakeLists.txt

@@ -143,6 +143,8 @@ set(FULL_NODE_SOURCE
   full-node-master.h
   full-node-master.hpp
   full-node-master.cpp
+  full-node-private-overlay.hpp
+  full-node-private-overlay.cpp
 
   net/download-block.hpp
   net/download-block.cpp

validator/downloaders/wait-block-state.cpp

@@ -109,12 +109,14 @@ void WaitBlockState::start() {
   } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result<td::BufferSlice> R) {
       if (R.is_error()) {
-        delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::start); }, td::Timestamp::in(0.1));
+        delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); },
+                     td::Timestamp::in(0.1));
       } else {
         td::actor::send_closure(SelfId, &WaitBlockState::got_proof_link, R.move_as_ok());
       }
     });
 
+    waiting_proof_link_ = true;
     td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), priority_,
                             std::move(P));
   } else if (prev_state_.is_null()) {
@@ -133,12 +135,14 @@ void WaitBlockState::start() {
   } else if (handle_->id().is_masterchain() && !handle_->inited_proof()) {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result<td::BufferSlice> R) {
       if (R.is_error()) {
-        delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::start); }, td::Timestamp::in(0.1));
+        delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof); },
+                     td::Timestamp::in(0.1));
       } else {
         td::actor::send_closure(SelfId, &WaitBlockState::got_proof, R.move_as_ok());
       }
     });
 
+    waiting_proof_ = true;
     td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_request, handle_->id(), priority_,
                             std::move(P));
   } else if (block_.is_null()) {
@@ -172,6 +176,9 @@ void WaitBlockState::got_prev_state(td::Ref<ShardState> state) {
 }
 
 void WaitBlockState::got_proof_link(td::BufferSlice data) {
+  if (!waiting_proof_link_) {
+    return;
+  }
   auto R = create_proof_link(handle_->id(), std::move(data));
   if (R.is_error()) {
     LOG(INFO) << "received bad proof link: " << R.move_as_error();
@@ -182,22 +189,25 @@ void WaitBlockState::got_proof_link(td::BufferSlice data) {
     if (R.is_ok()) {
       auto h = R.move_as_ok();
       CHECK(h->inited_prev());
-      td::actor::send_closure(SelfId, &WaitBlockState::start);
+      td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link);
     } else {
       LOG(INFO) << "received bad proof link: " << R.move_as_error();
-      td::actor::send_closure(SelfId, &WaitBlockState::start);
+      td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link);
     }
   });
   run_check_proof_link_query(handle_->id(), R.move_as_ok(), manager_, timeout_, std::move(P));
 }
 
 void WaitBlockState::got_proof(td::BufferSlice data) {
+  if (!waiting_proof_) {
+    return;
+  }
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
     if (R.is_ok()) {
-      td::actor::send_closure(SelfId, &WaitBlockState::start);
+      td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof);
     } else {
       LOG(INFO) << "received bad proof link: " << R.move_as_error();
-      td::actor::send_closure(SelfId, &WaitBlockState::start);
+      td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof);
     }
   });
   td::actor::send_closure(manager_, &ValidatorManager::validate_block_proof, handle_->id(), std::move(data),

validator/downloaders/wait-block-state.hpp

@@ -45,11 +45,9 @@ class WaitBlockState : public td::actor::Actor {
   void force_read_from_db();
 
   void start_up() override;
-  void got_block_handle(BlockHandle handle);
   void start();
   void got_state_from_db(td::Ref<ShardState> data);
   void got_state_from_static_file(td::Ref<ShardState> state, td::BufferSlice data);
-  void failed_to_get_state_from_db(td::Status reason);
   void got_prev_state(td::Ref<ShardState> state);
   void failed_to_get_prev_state(td::Status reason);
   void got_block_data(td::Ref<BlockData> data);
@@ -68,6 +66,22 @@ class WaitBlockState : public td::actor::Actor {
     priority_ = priority;
   }
 
+  // These two methods can be called from ValidatorManagerImpl::written_handle
+  void after_get_proof_link() {
+    if (!waiting_proof_link_) {
+      return;
+    }
+    waiting_proof_link_ = false;
+    start();
+  }
+  void after_get_proof() {
+    if (!waiting_proof_) {
+      return;
+    }
+    waiting_proof_ = false;
+    start();
+  }
+
  private:
   BlockHandle handle_;
 
@@ -81,6 +95,8 @@ class WaitBlockState : public td::actor::Actor {
   td::Ref<BlockData> block_;
 
   bool reading_from_db_ = false;
+  bool waiting_proof_link_ = false;
+  bool waiting_proof_ = false;
   td::Timestamp next_static_file_attempt_;
 
   td::PerfWarningTimer perf_timer_;

validator/full-node-private-overlay.cpp

@@ -0,0 +1,175 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+
+#include "full-node-private-overlay.hpp"
+#include "ton/ton-tl.hpp"
+#include "common/delay.h"
+
+namespace ton {
+
+namespace validator {
+
+namespace fullnode {
+
+void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) {
+  std::vector<BlockSignature> signatures;
+  for (auto &sig : query.signatures_) {
+    signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
+  }
+
+  BlockIdExt block_id = create_block_id(query.id_);
+  BlockBroadcast B{block_id,
+                   std::move(signatures),
+                   static_cast<UnixTime>(query.catchain_seqno_),
+                   static_cast<td::uint32>(query.validator_set_hash_),
+                   std::move(query.data_),
+                   std::move(query.proof_)};
+
+  auto P = td::PromiseCreator::lambda([](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      if (R.error().code() == ErrorCode::notready) {
+        LOG(DEBUG) << "dropped broadcast: " << R.move_as_error();
+      } else {
+        LOG(INFO) << "dropped broadcast: " << R.move_as_error();
+      }
+    }
+  });
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B),
+                          std::move(P));
+}
+
+void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_newShardBlockBroadcast &query) {
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block,
+                          create_block_id(query.block_->block_), query.block_->cc_seqno_,
+                          std::move(query.block_->data_));
+}
+
+void FullNodePrivateOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) {
+  auto B = fetch_tl_object<ton_api::tonNode_Broadcast>(std::move(broadcast), true);
+  if (B.is_error()) {
+    return;
+  }
+
+  ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); });
+}
+
+void FullNodePrivateOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) {
+  if (!inited_) {
+    return;
+  }
+  auto B = create_serialize_tl_object<ton_api::tonNode_newShardBlockBroadcast>(
+      create_tl_object<ton_api::tonNode_newShardBlock>(create_tl_block_id(block_id), cc_seqno, std::move(data)));
+  if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) {
+    td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_,
+                            local_id_.pubkey_hash(), 0, std::move(B));
+  } else {
+    td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_,
+                            local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
+  }
+}
+
+void FullNodePrivateOverlay::send_broadcast(BlockBroadcast broadcast) {
+  if (!inited_) {
+    return;
+  }
+  std::vector<tl_object_ptr<ton_api::tonNode_blockSignature>> sigs;
+  for (auto &sig : broadcast.signatures) {
+    sigs.emplace_back(create_tl_object<ton_api::tonNode_blockSignature>(sig.node, sig.signature.clone()));
+  }
+  auto B = create_serialize_tl_object<ton_api::tonNode_blockBroadcast>(
+      create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
+      broadcast.proof.clone(), broadcast.data.clone());
+  td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_,
+                          local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
+}
+
+void FullNodePrivateOverlay::start_up() {
+  std::sort(nodes_.begin(), nodes_.end());
+  nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end());
+
+  std::vector<td::Bits256> nodes;
+  for (const adnl::AdnlNodeIdShort &id : nodes_) {
+    nodes.push_back(id.bits256_value());
+  }
+  auto X = create_hash_tl_object<ton_api::tonNode_privateBlockOverlayId>(zero_state_file_hash_, std::move(nodes));
+  td::BufferSlice b{32};
+  b.as_slice().copy_from(as_slice(X));
+  overlay_id_full_ = overlay::OverlayIdFull{std::move(b)};
+  overlay_id_ = overlay_id_full_.compute_short_id();
+
+  try_init();
+}
+
+void FullNodePrivateOverlay::try_init() {
+  // Sometimes adnl id is added to validator engine later (or not at all)
+  td::actor::send_closure(
+      adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result<bool> R) {
+        if (R.is_ok() && R.ok()) {
+          td::actor::send_closure(SelfId, &FullNodePrivateOverlay::init);
+        } else {
+          delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodePrivateOverlay::try_init); },
+                       td::Timestamp::in(30.0));
+        }
+      });
+}
+
+void FullNodePrivateOverlay::init() {
+  LOG(FULL_NODE_INFO) << "Creating private block overlay for adnl id " << local_id_ << " : " << nodes_.size()
+                      << " nodes";
+  class Callback : public overlay::Overlays::Callback {
+   public:
+    void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override {
+    }
+    void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data,
+                       td::Promise<td::BufferSlice> promise) override {
+    }
+    void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override {
+      td::actor::send_closure(node_, &FullNodePrivateOverlay::receive_broadcast, src, std::move(data));
+    }
+    void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data,
+                         td::Promise<td::Unit> promise) override {
+    }
+    Callback(td::actor::ActorId<FullNodePrivateOverlay> node) : node_(node) {
+    }
+
+   private:
+    td::actor::ActorId<FullNodePrivateOverlay> node_;
+  };
+
+  overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(),
+                                     overlay::CertificateFlags::AllowFec | overlay::CertificateFlags::Trusted,
+                                     {}};
+  td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(),
+                          nodes_, std::make_unique<Callback>(actor_id(this)), rules);
+
+  td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_);
+  td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_);
+  inited_ = true;
+}
+
+void FullNodePrivateOverlay::tear_down() {
+  if (inited_) {
+    td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_);
+  }
+}
+
+}  // namespace fullnode
+
+}  // namespace validator
+
+}  // namespace ton

validator/full-node-private-overlay.hpp

@@ -0,0 +1,86 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+
+#include "full-node.h"
+
+namespace ton {
+
+namespace validator {
+
+namespace fullnode {
+
+class FullNodePrivateOverlay : public td::actor::Actor {
+ public:
+  void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query);
+  void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query);
+  template <class T>
+  void process_broadcast(PublicKeyHash, T &) {
+    VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast";
+  }
+  void receive_broadcast(PublicKeyHash src, td::BufferSlice query);
+
+  void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data);
+  void send_broadcast(BlockBroadcast broadcast);
+
+  void start_up() override;
+  void tear_down() override;
+
+  FullNodePrivateOverlay(adnl::AdnlNodeIdShort local_id, std::vector<adnl::AdnlNodeIdShort> nodes,
+                         FileHash zero_state_file_hash, FullNodeConfig config,
+                         td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
+                         td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2,
+                         td::actor::ActorId<overlay::Overlays> overlays,
+                         td::actor::ActorId<ValidatorManagerInterface> validator_manager)
+      : local_id_(local_id)
+      , nodes_(std::move(nodes))
+      , zero_state_file_hash_(zero_state_file_hash)
+      , config_(config)
+      , keyring_(keyring)
+      , adnl_(adnl)
+      , rldp_(rldp)
+      , rldp2_(rldp2)
+      , overlays_(overlays)
+      , validator_manager_(validator_manager) {
+  }
+
+ private:
+  adnl::AdnlNodeIdShort local_id_;
+  std::vector<adnl::AdnlNodeIdShort> nodes_;
+  FileHash zero_state_file_hash_;
+  FullNodeConfig config_;
+
+  td::actor::ActorId<keyring::Keyring> keyring_;
+  td::actor::ActorId<adnl::Adnl> adnl_;
+  td::actor::ActorId<rldp::Rldp> rldp_;
+  td::actor::ActorId<rldp2::Rldp> rldp2_;
+  td::actor::ActorId<overlay::Overlays> overlays_;
+  td::actor::ActorId<ValidatorManagerInterface> validator_manager_;
+
+  bool inited_ = false;
+  overlay::OverlayIdFull overlay_id_full_;
+  overlay::OverlayIdShort overlay_id_;
+
+  void try_init();
+  void init();
+};
+
+}  // namespace fullnode
+
+}  // namespace validator
+
+}  // namespace ton

validator/full-node.cpp

@@ -50,6 +50,7 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise<td::Unit> pr
   for (auto &shard : shards_) {
     td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_);
   }
+  create_private_block_overlay(key);
   promise.set_value(td::Unit());
 }
 
@@ -74,6 +75,7 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise<td::Unit> pr
   for (auto &shard : shards_) {
     td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_);
   }
+  private_block_overlays_.erase(key);
   promise.set_value(td::Unit());
 }
 
@@ -179,6 +181,10 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s
     VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard";
     return;
   }
+  if (!private_block_overlays_.empty()) {
+    td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_shard_block_info,
+                            block_id, cc_seqno, data.clone());
+  }
   td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data));
 }
 
@@ -188,6 +194,10 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) {
     VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard";
     return;
   }
+  if (!private_block_overlays_.empty()) {
+    td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_broadcast,
+                            broadcast.clone());
+  }
   td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast));
 }
 
@@ -289,6 +299,7 @@ void FullNodeImpl::got_key_block_proof(td::Ref<ProofLink> proof) {
 
   PublicKeyHash l = PublicKeyHash::zero();
   std::vector<PublicKeyHash> keys;
+  std::map<PublicKeyHash, adnl::AdnlNodeIdShort> current_validators;
   for (td::int32 i = -1; i <= 1; i++) {
     auto r = config->get_total_validator_set(i < 0 ? i : 1 - i);
     if (r.not_null()) {
@@ -299,9 +310,17 @@ void FullNodeImpl::got_key_block_proof(td::Ref<ProofLink> proof) {
         if (local_keys_.count(key)) {
           l = key;
         }
+        if (i == 1) {
+          current_validators[key] = adnl::AdnlNodeIdShort{el.addr.is_zero() ? key.bits256_value() : el.addr};
         }
       }
     }
+  }
+
+  if (current_validators != current_validators_) {
+    current_validators_ = std::move(current_validators);
+    update_private_block_overlays();
+  }
 
   if (keys == all_validators_) {
     return;
@@ -321,6 +340,7 @@ void FullNodeImpl::got_zero_block_state(td::Ref<ShardState> state) {
 
   PublicKeyHash l = PublicKeyHash::zero();
   std::vector<PublicKeyHash> keys;
+  std::map<PublicKeyHash, adnl::AdnlNodeIdShort> current_validators;
   for (td::int32 i = -1; i <= 1; i++) {
     auto r = m->get_total_validator_set(i < 0 ? i : 1 - i);
     if (r.not_null()) {
@@ -331,9 +351,17 @@ void FullNodeImpl::got_zero_block_state(td::Ref<ShardState> state) {
         if (local_keys_.count(key)) {
           l = key;
         }
+        if (i == 1) {
+          current_validators[key] = adnl::AdnlNodeIdShort{el.addr.is_zero() ? key.bits256_value() : el.addr};
         }
       }
     }
+  }
+
+  if (current_validators != current_validators_) {
+    current_validators_ = std::move(current_validators);
+    update_private_block_overlays();
+  }
 
   if (keys == all_validators_) {
     return;
@@ -456,6 +484,29 @@ void FullNodeImpl::start_up() {
                           std::make_unique<Callback>(actor_id(this)), std::move(P));
 }
 
+void FullNodeImpl::update_private_block_overlays() {
+  private_block_overlays_.clear();
+  if (local_keys_.empty()) {
+    return;
+  }
+  for (const auto &key : local_keys_) {
+    create_private_block_overlay(key);
+  }
+}
+
+void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) {
+  CHECK(local_keys_.count(key));
+  if (current_validators_.count(key)) {
+    std::vector<adnl::AdnlNodeIdShort> nodes;
+    for (const auto &p : current_validators_) {
+      nodes.push_back(p.second);
+    }
+    private_block_overlays_[key] = td::actor::create_actor<FullNodePrivateOverlay>(
+        "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_,
+        adnl_, rldp_, rldp2_, overlays_, validator_manager_);
+  }
+}
+
 FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash,
                            FullNodeConfig config, td::actor::ActorId<keyring::Keyring> keyring,
                            td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<rldp::Rldp> rldp,

validator/full-node.hpp

@@ -23,6 +23,7 @@
 //#include "ton-node-slave.h"
 #include "interfaces/proof.h"
 #include "interfaces/shard.h"
+#include "full-node-private-overlay.hpp"
 
 #include <map>
 #include <set>
@@ -111,9 +112,15 @@ class FullNodeImpl : public FullNode {
 
   PublicKeyHash sign_cert_by_;
   std::vector<PublicKeyHash> all_validators_;
+  std::map<PublicKeyHash, adnl::AdnlNodeIdShort> current_validators_;
 
   std::set<PublicKeyHash> local_keys_;
   FullNodeConfig config_;
+
+  std::map<PublicKeyHash, td::actor::ActorOwn<FullNodePrivateOverlay>> private_block_overlays_;
+
+  void update_private_block_overlays();
+  void create_private_block_overlay(PublicKeyHash key);
 };
 
 }  // namespace fullnode

validator/manager.cpp

@@ -1185,7 +1185,7 @@ void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise<td::Unit
 void ValidatorManagerImpl::written_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
   bool received = handle->received();
   bool inited_state = handle->received_state();
-  bool inited_proof = handle->id().is_masterchain() ? handle->inited_proof() : handle->inited_proof();
+  bool inited_proof = handle->id().is_masterchain() ? handle->inited_proof() : handle->inited_proof_link();
 
   if (handle->need_flush()) {
     handle->flush(actor_id(this), handle, std::move(promise));
@@ -1198,11 +1198,24 @@ void ValidatorManagerImpl::written_handle(BlockHandle handle, td::Promise<td::Un
       td::actor::send_closure(it->second.actor_, &WaitBlockData::force_read_from_db);
     }
   }
-  if (inited_state && inited_proof) {
+  if (inited_state) {
     auto it = wait_state_.find(handle->id());
     if (it != wait_state_.end()) {
       td::actor::send_closure(it->second.actor_, &WaitBlockState::force_read_from_db);
     }
+  } else {
+    if (handle->inited_proof_link()) {
+      auto it = wait_state_.find(handle->id());
+      if (it != wait_state_.end()) {
+        td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link);
+      }
+    }
+    if (handle->id().is_masterchain() && handle->inited_proof()) {
+      auto it = wait_state_.find(handle->id());
+      if (it != wait_state_.end()) {
+        td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof);
+      }
+    }
   }
 
   promise.set_value(td::Unit());

validator/validator-group.cpp

@@ -155,9 +155,9 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s
   td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, std::move(stats));
   auto block =
       block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref<BlockData>{};
+  bool send_broadcast = src == local_id_;
 
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = next_block_id, block, prev = prev_block_ids_,
+  auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), block_id = next_block_id, prev = prev_block_ids_,
-                                       sig_set, approve_sig_set,
                                        promise = std::move(promise)](td::Result<td::Unit> R) mutable {
     if (R.is_error()) {
       if (R.error().code() == ErrorCode::cancelled) {
@@ -166,14 +166,15 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s
       }
       LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error();
       td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block),
-                              std::move(prev), std::move(sig_set), std::move(approve_sig_set), std::move(promise));
+                              std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast,
+                              std::move(promise));
     } else {
       promise.set_value(R.move_as_ok());
     }
   });
 
   run_accept_block_query(next_block_id, std::move(block), prev_block_ids_, validator_set_, std::move(sig_set),
-                         std::move(approve_sig_set), src == local_id_, manager_, std::move(P));
+                         std::move(approve_sig_set), send_broadcast, manager_, std::move(P));
   prev_block_ids_ = std::vector<BlockIdExt>{next_block_id};
   cached_collated_block_ = nullptr;
   approved_candidates_cache_.clear();
@@ -181,21 +182,22 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s
 
 void ValidatorGroup::retry_accept_block_query(BlockIdExt block_id, td::Ref<BlockData> block,
                                               std::vector<BlockIdExt> prev, td::Ref<BlockSignatureSet> sig_set,
-                                              td::Ref<BlockSignatureSet> approve_sig_set,
+                                              td::Ref<BlockSignatureSet> approve_sig_set, bool send_broadcast,
                                               td::Promise<td::Unit> promise) {
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, block, prev, sig_set, approve_sig_set,
+  auto P = td::PromiseCreator::lambda(
-                                       promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+      [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::Unit> R) mutable {
         if (R.is_error()) {
           LOG_CHECK(R.error().code() == ErrorCode::timeout) << R.move_as_error();
           td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block),
-                              std::move(prev), std::move(sig_set), std::move(approve_sig_set), std::move(promise));
+                                  std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast,
+                                  std::move(promise));
         } else {
           promise.set_value(R.move_as_ok());
         }
       });
 
   run_accept_block_query(block_id, std::move(block), prev, validator_set_, std::move(sig_set),
-                         std::move(approve_sig_set), false, manager_, std::move(P));
+                         std::move(approve_sig_set), send_broadcast, manager_, std::move(P));
 }
 
 void ValidatorGroup::skip_round(td::uint32 round_id) {
@@ -347,7 +349,7 @@ void ValidatorGroup::start(std::vector<BlockIdExt> prev, BlockIdExt min_masterch
     auto block =
         p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref<BlockData>{};
     retry_accept_block_query(next_block_id, std::move(block), prev_block_ids_, std::move(p.sigs),
-                             std::move(p.approve_sigs), std::move(p.promise));
+                             std::move(p.approve_sigs), false, std::move(p.promise));
     prev_block_ids_ = std::vector<BlockIdExt>{next_block_id};
   }
   postponed_accept_.clear();

validator/validator-group.hpp

@@ -43,7 +43,7 @@ class ValidatorGroup : public td::actor::Actor {
   void skip_round(td::uint32 round);
   void retry_accept_block_query(BlockIdExt block_id, td::Ref<BlockData> block, std::vector<BlockIdExt> prev,
                                 td::Ref<BlockSignatureSet> sigs, td::Ref<BlockSignatureSet> approve_sigs,
-                                td::Promise<td::Unit> promise);
+                                bool send_broadcast, td::Promise<td::Unit> promise);
   void get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash,
                               FileHash collated_data_file_hash, td::Promise<BlockCandidate> promise);
   BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const;