Prepare transition between old and new private overlays

validator/full-node-private-overlay-v2.cpp

@@ -239,7 +239,7 @@ void FullNodePrivateOverlayV2::get_stats_extra(td::Promise<std::string> promise)
   promise.set_result(td::json_encode<std::string>(td::ToJson(*res), true));
 }
 
-td::actor::ActorId<FullNodePrivateOverlayV2> FullNodePrivateBlockOverlays::choose_overlay(ShardIdFull shard) {
+td::actor::ActorId<FullNodePrivateOverlayV2> FullNodePrivateBlockOverlaysV2::choose_overlay(ShardIdFull shard) {
   for (auto &p : id_to_overlays_) {
     auto &overlays = p.second.overlays_;
     ShardIdFull cur_shard = shard;
@@ -257,7 +257,7 @@ td::actor::ActorId<FullNodePrivateOverlayV2> FullNodePrivateBlockOverlays::choos
   return {};
 }
 
-void FullNodePrivateBlockOverlays::update_overlays(
+void FullNodePrivateBlockOverlaysV2::update_overlays(
     td::Ref<MasterchainState> state, std::set<adnl::AdnlNodeIdShort> my_adnl_ids, const FileHash &zero_state_file_hash,
     const td::actor::ActorId<keyring::Keyring> &keyring, const td::actor::ActorId<adnl::Adnl> &adnl,
     const td::actor::ActorId<rldp::Rldp> &rldp, const td::actor::ActorId<rldp2::Rldp> &rldp2,
@@ -274,10 +274,10 @@ void FullNodePrivateBlockOverlays::update_overlays(
   struct OverlayInfo {
     std::vector<adnl::AdnlNodeIdShort> nodes, senders;
   };
-  std::map<ShardIdFull, OverlayInfo> pverlay_infos;
+  std::map<ShardIdFull, OverlayInfo> overlay_infos;
 
   // Masterchain overlay: all validators + collators
-  OverlayInfo &mc_overlay = pverlay_infos[ShardIdFull(masterchainId)];
+  OverlayInfo &mc_overlay = overlay_infos[ShardIdFull(masterchainId)];
   for (const auto &x : all_validators->export_vector()) {
     td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr;
     mc_overlay.nodes.emplace_back(addr);
@@ -311,14 +311,14 @@ void FullNodePrivateBlockOverlays::update_overlays(
     auto val_set = state->get_validator_set(shard);
     td::uint32 min_split = state->monitor_min_split_depth(shard.workchain);
     OverlayInfo &overlay =
-        pverlay_infos[shard_prefix_length(shard) <= min_split ? shard : shard_prefix(shard, min_split)];
+        overlay_infos[shard_prefix_length(shard) <= min_split ? shard : shard_prefix(shard, min_split)];
     for (const auto &x : val_set->export_vector()) {
       td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr;
       overlay.nodes.emplace_back(addr);
       overlay.senders.emplace_back(addr);
     }
   }
-  for (auto &p : pverlay_infos) {
+  for (auto &p : overlay_infos) {
     ShardIdFull shard = p.first;
     OverlayInfo &overlay = p.second;
     if (!shard.is_masterchain()) {
@@ -338,7 +338,7 @@ void FullNodePrivateBlockOverlays::update_overlays(
   std::map<adnl::AdnlNodeIdShort, Overlays> old_private_block_overlays = std::move(id_to_overlays_);
   id_to_overlays_.clear();
 
-  for (const auto &p : pverlay_infos) {
+  for (const auto &p : overlay_infos) {
     ShardIdFull shard = p.first;
     const OverlayInfo &new_overlay_info = p.second;
     for (adnl::AdnlNodeIdShort local_id : new_overlay_info.nodes) {
@@ -356,8 +356,9 @@ void FullNodePrivateBlockOverlays::update_overlays(
         new_overlay.senders_ = new_overlay_info.senders;
         new_overlay.is_sender_ = std::binary_search(new_overlay.senders_.begin(), new_overlay.senders_.end(), local_id);
         new_overlay.overlay_ = td::actor::create_actor<FullNodePrivateOverlayV2>(
-            "BlocksPrivateOverlay", local_id, shard, new_overlay.nodes_, new_overlay.senders_, zero_state_file_hash,
+            PSTRING() << "BlocksPrivateOverlay" << shard.to_str(), local_id, shard, new_overlay.nodes_,
-            keyring, adnl, rldp, rldp2, overlays, validator_manager, full_node);
+            new_overlay.senders_, zero_state_file_hash, keyring, adnl, rldp, rldp2, overlays, validator_manager,
+            full_node);
       }
     }
   }
@@ -370,9 +371,14 @@ void FullNodePrivateBlockOverlays::update_overlays(
       }
       td::actor::ActorId<FullNodePrivateOverlayV2> id = x.second.overlay_.release();
       delay_action([id = std::move(id)]() { td::actor::send_closure(id, &FullNodePrivateOverlayV2::destroy); },
-                   td::Timestamp::in(60.0));
+                   td::Timestamp::in(30.0));
     }
   }
 }
 
+void FullNodePrivateBlockOverlaysV2::destroy_overlays() {
+  id_to_overlays_.clear();
+}
+
+
 }  // namespace ton::validator::fullnode

validator/full-node-private-overlay-v2.hpp

@@ -96,7 +96,7 @@ class FullNodePrivateOverlayV2 : public td::actor::Actor {
   void get_stats_extra(td::Promise<std::string> promise);
 };
 
-class FullNodePrivateBlockOverlays {
+class FullNodePrivateBlockOverlaysV2 {
  public:
   td::actor::ActorId<FullNodePrivateOverlayV2> choose_overlay(ShardIdFull shard);
   void update_overlays(td::Ref<MasterchainState> state, std::set<adnl::AdnlNodeIdShort> my_adnl_ids,
@@ -106,6 +106,7 @@ class FullNodePrivateBlockOverlays {
                        const td::actor::ActorId<overlay::Overlays>& overlays,
                        const td::actor::ActorId<ValidatorManagerInterface>& validator_manager,
                        const td::actor::ActorId<FullNode>& full_node);
+  void destroy_overlays();
 
  private:
   struct Overlays {

validator/full-node.cpp

@@ -37,7 +37,7 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise<td::Unit> pr
   }
 
   local_keys_.insert(key);
-  // create_private_block_overlay(key);
+  create_private_block_overlay(key);
   for (auto &p : custom_overlays_) {
     update_custom_overlay(p.second);
   }
@@ -67,7 +67,7 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise<td::Unit> pr
     return;
   }
   local_keys_.erase(key);
-  // private_block_overlays_.erase(key);
+  private_block_overlays_.erase(key);
   for (auto &p : custom_overlays_) {
     update_custom_overlay(p.second);
   }
@@ -151,9 +151,9 @@ void FullNodeImpl::set_config(FullNodeConfig config) {
       td::actor::send_closure(s.second.actor, &FullNodeShard::set_config, config);
     }
   }
-  /*for (auto& overlay : private_block_overlays_) {
+  for (auto& overlay : private_block_overlays_) {
     td::actor::send_closure(overlay.second, &FullNodePrivateBlockOverlay::set_config, config);
-  }*/
+  }
   for (auto& overlay : custom_overlays_) {
     for (auto &actor : overlay.second.actors_) {
       td::actor::send_closure(actor.second, &FullNodeCustomOverlay::set_config, config);
@@ -286,8 +286,12 @@ void FullNodeImpl::update_shard_configuration(td::Ref<MasterchainState> state, s
       my_adnl_ids.insert(it->second);
     }
   }
-  private_block_overlays_.update_overlays(state, std::move(my_adnl_ids), zero_state_file_hash_, keyring_, adnl_, rldp_,
+  if (use_old_private_overlays_) {
-                                          rldp2_, overlays_, validator_manager_, actor_id(this));
+    private_block_overlays_v2_.destroy_overlays();
+  } else {
+    private_block_overlays_v2_.update_overlays(state, std::move(my_adnl_ids), zero_state_file_hash_, keyring_, adnl_,
+                                               rldp_, rldp2_, overlays_, validator_manager_, actor_id(this));
+  }
 }
 
 void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) {
@@ -340,11 +344,11 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s
     VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard";
     return;
   }
-  /*if (!private_block_overlays_.empty()) {
+  if (!private_block_overlays_.empty()) {
     td::actor::send_closure(private_block_overlays_.begin()->second,
                             &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone());
-  }*/
+  }
-  auto private_overlay = private_block_overlays_.choose_overlay(ShardIdFull(masterchainId));
+  auto private_overlay = private_block_overlays_v2_.choose_overlay(ShardIdFull(masterchainId));
   if (!private_overlay.empty()) {
     td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_shard_block_info, block_id, cc_seqno,
                             data.clone());
@@ -360,11 +364,11 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se
     VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard";
     return;
   }
-  /*if (!private_block_overlays_.empty()) {
+  if (!private_block_overlays_.empty()) {
     td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate,
                             block_id, cc_seqno, validator_set_hash, data.clone());
-  }*/
+  }
-  auto private_overlay = private_block_overlays_.choose_overlay(block_id.shard_full());
+  auto private_overlay = private_block_overlays_v2_.choose_overlay(block_id.shard_full());
   if (!private_overlay.empty()) {
     td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_block_candidate, block_id, cc_seqno,
                             validator_set_hash, data.clone());
@@ -385,11 +389,11 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays
     VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard";
     return;
   }
-  /*if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) {
+  if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) {
     td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast,
                             broadcast.clone());
-  }*/
+  }
-  auto private_overlay = private_block_overlays_.choose_overlay(broadcast.block_id.shard_full());
+  auto private_overlay = private_block_overlays_v2_.choose_overlay(broadcast.block_id.shard_full());
   if (!private_overlay.empty()) {
     td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_broadcast, broadcast.clone());
   }
@@ -550,7 +554,7 @@ void FullNodeImpl::got_key_block_state(td::Ref<ShardState> state) {
     }
   }
 
-  // set_private_block_overlays_enable_compression(m->get_consensus_config().proto_version >= 3);
+  set_private_block_overlays_enable_compression(m->get_consensus_config().proto_version >= 3);
 
   if (current_validators != current_validators_) {
     current_validators_ = std::move(current_validators);
@@ -703,16 +707,17 @@ void FullNodeImpl::update_private_overlays() {
   for (auto &p : custom_overlays_) {
     update_custom_overlay(p.second);
   }
-  /*private_block_overlays_.clear();
+
+  private_block_overlays_.clear();
   if (local_keys_.empty()) {
     return;
   }
   for (const auto &key : local_keys_) {
     create_private_block_overlay(key);
-  }*/
+  }
 }
 
-/*void FullNodeImpl::set_private_block_overlays_enable_compression(bool value) {
+void FullNodeImpl::set_private_block_overlays_enable_compression(bool value) {
   if (private_block_overlays_enable_compression_ == value) {
     return;
   }
@@ -723,6 +728,9 @@ void FullNodeImpl::update_private_overlays() {
 }
 
 void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) {
+  if (!use_old_private_overlays_) {
+    return;
+  }
   CHECK(local_keys_.count(key));
   if (current_validators_.count(key)) {
     std::vector<adnl::AdnlNodeIdShort> nodes;
@@ -734,7 +742,7 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) {
         private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_,
         actor_id(this));
   }
-}*/
+}
 
 void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) {
   auto old_actors = std::move(overlay.actors_);

validator/full-node.h

@@ -75,8 +75,9 @@ class FullNode : public td::actor::Actor {
   virtual void add_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0;
   virtual void del_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0;
 
-  virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at,
+  virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key,
-                                              td::uint32 max_size, td::Promise<td::BufferSlice> promise) = 0;
+                                              td::uint32 expiry_at, td::uint32 max_size,
+                                              td::Promise<td::BufferSlice> promise) = 0;
   virtual void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key,
                                                 std::shared_ptr<ton::overlay::Certificate> cert,
                                                 td::Promise<td::Unit> promise) = 0;

validator/full-node.hpp

@@ -144,14 +144,14 @@ class FullNodeImpl : public FullNode {
   td::Promise<td::Unit> started_promise_;
   FullNodeConfig config_;
 
-  // TODO: Decide what to do with old private overlays. Maybe use old or new depending on some flag in config.
+  // Private overlays:
-  /*
+  // Old overlays - one overlay for all validators
-  std::map<PublicKeyHash, td::actor::ActorOwn<FullNodePrivateOverlay>> private_block_overlays_;
+  // New overlays (v2) - overlay per shard (monitor_min_split depth).
+  bool use_old_private_overlays_ = false;  // TODO: set from config
+  std::map<PublicKeyHash, td::actor::ActorOwn<FullNodePrivateBlockOverlay>> private_block_overlays_;
   bool private_block_overlays_enable_compression_ = false;
-  void set_private_block_overlays_enable_compression(bool value);
-  void create_private_block_overlay(PublicKeyHash key);
-  */
   bool broadcast_block_candidates_in_public_overlay_ = false;
+  FullNodePrivateBlockOverlaysV2 private_block_overlays_v2_;
 
   struct CustomOverlayInfo {
     CustomOverlayParams params_;
@@ -162,13 +162,12 @@ class FullNodeImpl : public FullNode {
   std::queue<BlockIdExt> custom_overlays_sent_broadcasts_lru_;
 
   void update_private_overlays();
-  // void set_private_block_overlays_enable_compression(bool value);
+  void set_private_block_overlays_enable_compression(bool value);
-  // void create_private_block_overlay(PublicKeyHash key);
+  void create_private_block_overlay(PublicKeyHash key);
   void update_custom_overlay(CustomOverlayInfo& overlay);
   void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast);
   void send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt& block_id, CatchainSeqno cc_seqno,
                                                          td::uint32 validator_set_hash, const td::BufferSlice& data);
-  FullNodePrivateBlockOverlays private_block_overlays_;
 };
 
 }  // namespace fullnode

validator/manager.cpp

@@ -504,6 +504,10 @@ void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSl
   if (!started_) {
     return;
   }
+  if (!need_monitor(block_id.shard_full())) {
+    VLOG(VALIDATOR_DEBUG) << "dropping block candidate broadcast: not monitoring shard";
+    return;
+  }
   add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)});
 }