Add special overlay for validators for block broadcasting (#842)

* Private overlay for broadcasting blocks

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>

adnl/adnl-peer-table.hpp

@@ -77,6 +77,10 @@ class AdnlPeerTableImpl : public AdnlPeerTable {
                         td::actor::ActorId<AdnlChannel> channel) override;
   void unregister_channel(AdnlChannelIdShort id) override;
 
+  void check_id_exists(AdnlNodeIdShort id, td::Promise<bool> promise) override {
+    promise.set_value(local_ids_.count(id));
+  }
+
   void write_new_addr_list_to_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, AdnlDbItem node,
                                  td::Promise<td::Unit> promise) override;
   void get_addr_list_from_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id,

adnl/adnl.h

@@ -97,6 +97,8 @@ class Adnl : public AdnlSenderInterface {
   virtual void add_id_ex(AdnlNodeIdFull id, AdnlAddressList addr_list, td::uint8 cat, td::uint32 mode) = 0;
   virtual void del_id(AdnlNodeIdShort id, td::Promise<td::Unit> promise) = 0;
 
+  virtual void check_id_exists(AdnlNodeIdShort id, td::Promise<bool> promise) = 0;
+
   // subscribe to (some) messages(+queries) to this local id
   virtual void subscribe(AdnlNodeIdShort dst, std::string prefix, std::unique_ptr<Callback> callback) = 0;
   virtual void unsubscribe(AdnlNodeIdShort dst, std::string prefix) = 0;

tl/generate/scheme/ton_api.tl

@@ -394,6 +394,8 @@ tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast;
 
 tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId;
 
+tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId;
+
 tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool = tonNode.KeyBlocks;
 
 ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId;

ton/ton-types.h

@@ -353,6 +353,14 @@ struct BlockBroadcast {
   td::uint32 validator_set_hash;
   td::BufferSlice data;
   td::BufferSlice proof;
+
+  BlockBroadcast clone() const {
+    std::vector<BlockSignature> new_signatures;
+    for (const BlockSignature& s : signatures) {
+      new_signatures.emplace_back(s.node, s.signature.clone());
+    }
+    return {block_id, std::move(new_signatures), catchain_seqno, validator_set_hash, data.clone(), proof.clone()};
+  }
 };
 
 struct Ed25519_PrivateKey {

validator/CMakeLists.txt

@@ -143,6 +143,8 @@ set(FULL_NODE_SOURCE
   full-node-master.h
   full-node-master.hpp
   full-node-master.cpp
+  full-node-private-overlay.hpp
+  full-node-private-overlay.cpp
 
   net/download-block.hpp
   net/download-block.cpp

validator/full-node-private-overlay.cpp

@@ -0,0 +1,175 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+
+#include "full-node-private-overlay.hpp"
+#include "ton/ton-tl.hpp"
+#include "common/delay.h"
+
+namespace ton {
+
+namespace validator {
+
+namespace fullnode {
+
+void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) {
+  std::vector<BlockSignature> signatures;
+  for (auto &sig : query.signatures_) {
+    signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
+  }
+
+  BlockIdExt block_id = create_block_id(query.id_);
+  BlockBroadcast B{block_id,
+                   std::move(signatures),
+                   static_cast<UnixTime>(query.catchain_seqno_),
+                   static_cast<td::uint32>(query.validator_set_hash_),
+                   std::move(query.data_),
+                   std::move(query.proof_)};
+
+  auto P = td::PromiseCreator::lambda([](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      if (R.error().code() == ErrorCode::notready) {
+        LOG(DEBUG) << "dropped broadcast: " << R.move_as_error();
+      } else {
+        LOG(INFO) << "dropped broadcast: " << R.move_as_error();
+      }
+    }
+  });
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B),
+                          std::move(P));
+}
+
+void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_newShardBlockBroadcast &query) {
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block,
+                          create_block_id(query.block_->block_), query.block_->cc_seqno_,
+                          std::move(query.block_->data_));
+}
+
+void FullNodePrivateOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) {
+  auto B = fetch_tl_object<ton_api::tonNode_Broadcast>(std::move(broadcast), true);
+  if (B.is_error()) {
+    return;
+  }
+
+  ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); });
+}
+
+void FullNodePrivateOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) {
+  if (!inited_) {
+    return;
+  }
+  auto B = create_serialize_tl_object<ton_api::tonNode_newShardBlockBroadcast>(
+      create_tl_object<ton_api::tonNode_newShardBlock>(create_tl_block_id(block_id), cc_seqno, std::move(data)));
+  if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) {
+    td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_,
+                            local_id_.pubkey_hash(), 0, std::move(B));
+  } else {
+    td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_,
+                            local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
+  }
+}
+
+void FullNodePrivateOverlay::send_broadcast(BlockBroadcast broadcast) {
+  if (!inited_) {
+    return;
+  }
+  std::vector<tl_object_ptr<ton_api::tonNode_blockSignature>> sigs;
+  for (auto &sig : broadcast.signatures) {
+    sigs.emplace_back(create_tl_object<ton_api::tonNode_blockSignature>(sig.node, sig.signature.clone()));
+  }
+  auto B = create_serialize_tl_object<ton_api::tonNode_blockBroadcast>(
+      create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
+      broadcast.proof.clone(), broadcast.data.clone());
+  td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_,
+                          local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
+}
+
+void FullNodePrivateOverlay::start_up() {
+  std::sort(nodes_.begin(), nodes_.end());
+  nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end());
+
+  std::vector<td::Bits256> nodes;
+  for (const adnl::AdnlNodeIdShort &id : nodes_) {
+    nodes.push_back(id.bits256_value());
+  }
+  auto X = create_hash_tl_object<ton_api::tonNode_privateBlockOverlayId>(zero_state_file_hash_, std::move(nodes));
+  td::BufferSlice b{32};
+  b.as_slice().copy_from(as_slice(X));
+  overlay_id_full_ = overlay::OverlayIdFull{std::move(b)};
+  overlay_id_ = overlay_id_full_.compute_short_id();
+
+  try_init();
+}
+
+void FullNodePrivateOverlay::try_init() {
+  // Sometimes adnl id is added to validator engine later (or not at all)
+  td::actor::send_closure(
+      adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result<bool> R) {
+        if (R.is_ok() && R.ok()) {
+          td::actor::send_closure(SelfId, &FullNodePrivateOverlay::init);
+        } else {
+          delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodePrivateOverlay::try_init); },
+                       td::Timestamp::in(30.0));
+        }
+      });
+}
+
+void FullNodePrivateOverlay::init() {
+  LOG(FULL_NODE_INFO) << "Creating private block overlay for adnl id " << local_id_ << " : " << nodes_.size()
+                      << " nodes";
+  class Callback : public overlay::Overlays::Callback {
+   public:
+    void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override {
+    }
+    void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data,
+                       td::Promise<td::BufferSlice> promise) override {
+    }
+    void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override {
+      td::actor::send_closure(node_, &FullNodePrivateOverlay::receive_broadcast, src, std::move(data));
+    }
+    void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data,
+                         td::Promise<td::Unit> promise) override {
+    }
+    Callback(td::actor::ActorId<FullNodePrivateOverlay> node) : node_(node) {
+    }
+
+   private:
+    td::actor::ActorId<FullNodePrivateOverlay> node_;
+  };
+
+  overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(),
+                                     overlay::CertificateFlags::AllowFec | overlay::CertificateFlags::Trusted,
+                                     {}};
+  td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(),
+                          nodes_, std::make_unique<Callback>(actor_id(this)), rules);
+
+  td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_);
+  td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_);
+  inited_ = true;
+}
+
+void FullNodePrivateOverlay::tear_down() {
+  if (inited_) {
+    td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_);
+  }
+}
+
+}  // namespace fullnode
+
+}  // namespace validator
+
+}  // namespace ton

validator/full-node-private-overlay.hpp

@@ -0,0 +1,86 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+
+#include "full-node.h"
+
+namespace ton {
+
+namespace validator {
+
+namespace fullnode {
+
+class FullNodePrivateOverlay : public td::actor::Actor {
+ public:
+  void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query);
+  void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query);
+  template <class T>
+  void process_broadcast(PublicKeyHash, T &) {
+    VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast";
+  }
+  void receive_broadcast(PublicKeyHash src, td::BufferSlice query);
+
+  void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data);
+  void send_broadcast(BlockBroadcast broadcast);
+
+  void start_up() override;
+  void tear_down() override;
+
+  FullNodePrivateOverlay(adnl::AdnlNodeIdShort local_id, std::vector<adnl::AdnlNodeIdShort> nodes,
+                         FileHash zero_state_file_hash, FullNodeConfig config,
+                         td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
+                         td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2,
+                         td::actor::ActorId<overlay::Overlays> overlays,
+                         td::actor::ActorId<ValidatorManagerInterface> validator_manager)
+      : local_id_(local_id)
+      , nodes_(std::move(nodes))
+      , zero_state_file_hash_(zero_state_file_hash)
+      , config_(config)
+      , keyring_(keyring)
+      , adnl_(adnl)
+      , rldp_(rldp)
+      , rldp2_(rldp2)
+      , overlays_(overlays)
+      , validator_manager_(validator_manager) {
+  }
+
+ private:
+  adnl::AdnlNodeIdShort local_id_;
+  std::vector<adnl::AdnlNodeIdShort> nodes_;
+  FileHash zero_state_file_hash_;
+  FullNodeConfig config_;
+
+  td::actor::ActorId<keyring::Keyring> keyring_;
+  td::actor::ActorId<adnl::Adnl> adnl_;
+  td::actor::ActorId<rldp::Rldp> rldp_;
+  td::actor::ActorId<rldp2::Rldp> rldp2_;
+  td::actor::ActorId<overlay::Overlays> overlays_;
+  td::actor::ActorId<ValidatorManagerInterface> validator_manager_;
+
+  bool inited_ = false;
+  overlay::OverlayIdFull overlay_id_full_;
+  overlay::OverlayIdShort overlay_id_;
+
+  void try_init();
+  void init();
+};
+
+}  // namespace fullnode
+
+}  // namespace validator
+
+}  // namespace ton

validator/full-node.cpp

@@ -50,6 +50,7 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise<td::Unit> pr
   for (auto &shard : shards_) {
     td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_);
   }
+  create_private_block_overlay(key);
   promise.set_value(td::Unit());
 }
 
@@ -74,6 +75,7 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise<td::Unit> pr
   for (auto &shard : shards_) {
     td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_);
   }
+  private_block_overlays_.erase(key);
   promise.set_value(td::Unit());
 }
 
@@ -179,6 +181,10 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s
     VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard";
     return;
   }
+  if (!private_block_overlays_.empty()) {
+    td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_shard_block_info,
+                            block_id, cc_seqno, data.clone());
+  }
   td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data));
 }
 
@@ -188,6 +194,10 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) {
     VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard";
     return;
   }
+  if (!private_block_overlays_.empty()) {
+    td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_broadcast,
+                            broadcast.clone());
+  }
   td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast));
 }
 
@@ -289,6 +299,7 @@ void FullNodeImpl::got_key_block_proof(td::Ref<ProofLink> proof) {
 
   PublicKeyHash l = PublicKeyHash::zero();
   std::vector<PublicKeyHash> keys;
+  std::map<PublicKeyHash, adnl::AdnlNodeIdShort> current_validators;
   for (td::int32 i = -1; i <= 1; i++) {
     auto r = config->get_total_validator_set(i < 0 ? i : 1 - i);
     if (r.not_null()) {
@@ -299,9 +310,17 @@ void FullNodeImpl::got_key_block_proof(td::Ref<ProofLink> proof) {
         if (local_keys_.count(key)) {
           l = key;
         }
+        if (i == 1) {
+          current_validators[key] = adnl::AdnlNodeIdShort{el.addr.is_zero() ? key.bits256_value() : el.addr};
         }
       }
     }
+  }
+
+  if (current_validators != current_validators_) {
+    current_validators_ = std::move(current_validators);
+    update_private_block_overlays();
+  }
 
   if (keys == all_validators_) {
     return;
@@ -321,6 +340,7 @@ void FullNodeImpl::got_zero_block_state(td::Ref<ShardState> state) {
 
   PublicKeyHash l = PublicKeyHash::zero();
   std::vector<PublicKeyHash> keys;
+  std::map<PublicKeyHash, adnl::AdnlNodeIdShort> current_validators;
   for (td::int32 i = -1; i <= 1; i++) {
     auto r = m->get_total_validator_set(i < 0 ? i : 1 - i);
     if (r.not_null()) {
@@ -331,9 +351,17 @@ void FullNodeImpl::got_zero_block_state(td::Ref<ShardState> state) {
         if (local_keys_.count(key)) {
           l = key;
         }
+        if (i == 1) {
+          current_validators[key] = adnl::AdnlNodeIdShort{el.addr.is_zero() ? key.bits256_value() : el.addr};
         }
       }
     }
+  }
+
+  if (current_validators != current_validators_) {
+    current_validators_ = std::move(current_validators);
+    update_private_block_overlays();
+  }
 
   if (keys == all_validators_) {
     return;
@@ -456,6 +484,29 @@ void FullNodeImpl::start_up() {
                           std::make_unique<Callback>(actor_id(this)), std::move(P));
 }
 
+void FullNodeImpl::update_private_block_overlays() {
+  private_block_overlays_.clear();
+  if (local_keys_.empty()) {
+    return;
+  }
+  for (const auto &key : local_keys_) {
+    create_private_block_overlay(key);
+  }
+}
+
+void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) {
+  CHECK(local_keys_.count(key));
+  if (current_validators_.count(key)) {
+    std::vector<adnl::AdnlNodeIdShort> nodes;
+    for (const auto &p : current_validators_) {
+      nodes.push_back(p.second);
+    }
+    private_block_overlays_[key] = td::actor::create_actor<FullNodePrivateOverlay>(
+        "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_,
+        adnl_, rldp_, rldp2_, overlays_, validator_manager_);
+  }
+}
+
 FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash,
                            FullNodeConfig config, td::actor::ActorId<keyring::Keyring> keyring,
                            td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<rldp::Rldp> rldp,

validator/full-node.hpp

@@ -23,6 +23,7 @@
 //#include "ton-node-slave.h"
 #include "interfaces/proof.h"
 #include "interfaces/shard.h"
+#include "full-node-private-overlay.hpp"
 
 #include <map>
 #include <set>
@@ -111,9 +112,15 @@ class FullNodeImpl : public FullNode {
 
   PublicKeyHash sign_cert_by_;
   std::vector<PublicKeyHash> all_validators_;
+  std::map<PublicKeyHash, adnl::AdnlNodeIdShort> current_validators_;
 
   std::set<PublicKeyHash> local_keys_;
   FullNodeConfig config_;
+
+  std::map<PublicKeyHash, td::actor::ActorOwn<FullNodePrivateOverlay>> private_block_overlays_;
+
+  void update_private_block_overlays();
+  void create_private_block_overlay(PublicKeyHash key);
 };
 
 }  // namespace fullnode