Dynamic catchain delays, state serialization improvements (#1140)

* Validator improvements

* Fix cancelling state serialization
* Disable state serializer on all mainnet validators
* Flag --catchain-max-block-delay-slow

* Set default catchain-max-block-delay to 0.4, delay-slow to 1.0

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>

validator/impl/shard.cpp

@@ -44,6 +44,7 @@ ShardStateQ::ShardStateQ(const ShardStateQ& other)
     , root(other.root)
     , lt(other.lt)
     , utime(other.utime)
+    , global_id_(other.global_id_)
     , before_split_(other.before_split_)
     , fake_split_(other.fake_split_)
     , fake_merge_(other.fake_merge_) {
@@ -121,6 +122,7 @@ td::Status ShardStateQ::init() {
   }
   lt = info.gen_lt;
   utime = info.gen_utime;
+  global_id_ = info.global_id;
   before_split_ = info.before_split;
   block::ShardId id{info.shard_id};
   ton::BlockId hdr_id{ton::ShardIdFull(id), info.seq_no};

validator/impl/shard.hpp

@@ -38,6 +38,7 @@ class ShardStateQ : virtual public ShardState {
   Ref<vm::Cell> root;
   LogicalTime lt{0};
   UnixTime utime{0};
+  td::int32 global_id_{0};
   bool before_split_{false};
   bool fake_split_{false};
   bool fake_merge_{false};
@@ -81,6 +82,9 @@ class ShardStateQ : virtual public ShardState {
   LogicalTime get_logical_time() const override {
     return lt;
   }
+  td::int32 get_global_id() const override {
+    return global_id_;
+  }
   td::optional<BlockIdExt> get_master_ref() const override {
     return master_ref;
   }

validator/interfaces/shard.h

@@ -39,6 +39,7 @@ class ShardState : public td::CntObject {
 
   virtual UnixTime get_unix_time() const = 0;
   virtual LogicalTime get_logical_time() const = 0;
+  virtual td::int32 get_global_id() const = 0;
   virtual ShardIdFull get_shard() const = 0;
   virtual BlockSeqno get_seqno() const = 0;
   virtual BlockIdExt get_block_id() const = 0;

validator/manager.cpp

@@ -2057,7 +2057,6 @@ void ValidatorManagerImpl::update_shards() {
     }
   }
 
-  bool validating_masterchain = false;
   if (allow_validate_) {
     for (auto &desc : new_shards) {
       auto shard = desc.first;
@@ -2074,9 +2073,6 @@ void ValidatorManagerImpl::update_shards() {
       auto validator_id = get_validator(shard, val_set);
 
       if (!validator_id.is_zero()) {
-        if (shard.is_masterchain()) {
-          validating_masterchain = true;
-        }
         auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts);
 
         if (force_recover) {
@@ -2171,16 +2167,14 @@ void ValidatorManagerImpl::update_shards() {
       td::actor::send_closure(SelfId, &ValidatorManagerImpl::written_destroyed_validator_sessions, std::move(gc));
     });
     td::actor::send_closure(db_, &Db::update_destroyed_validator_sessions, gc_list_, std::move(P));
-
-    if (!serializer_.empty()) {
-      td::actor::send_closure(
-          serializer_, &AsyncStateSerializer::auto_disable_serializer,
-          validating_masterchain &&
-              last_masterchain_state_->get_validator_set(ShardIdFull{masterchainId})->export_vector().size() * 2 <=
-                  last_masterchain_state_->get_total_validator_set(0)->export_vector().size());
-    }
   }
-}  // namespace validator
+
+  if (!serializer_.empty()) {
+    td::actor::send_closure(
+        serializer_, &AsyncStateSerializer::auto_disable_serializer,
+        !validator_groups_.empty() && last_masterchain_state_->get_global_id() == -239);  // mainnet only
+  }
+}
 
 void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vector<td::actor::ActorId<ValidatorGroup>> list) {
   for (auto &v : list) {

validator/state-serializer.cpp

@@ -378,6 +378,7 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) {
 
 void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardState> state,
                                            std::shared_ptr<vm::CellDbReader> cell_db_reader) {
+  next_idx_++;
   if (!opts_->get_state_serializer_enabled() || auto_disabled_) {
     success_handler();
     return;
@@ -406,7 +407,6 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardStat
   });
   td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(),
                           masterchain_handle_->id(), write_data, std::move(P));
-  next_idx_++;
 }
 
 void AsyncStateSerializer::fail_handler(td::Status reason) {

validator/validator-group.cpp

@@ -348,10 +348,12 @@ void ValidatorGroup::create_session() {
                   << ".",
         allow_unsafe_self_blocks_resync_);
   }
-  if (opts_->get_catchain_max_block_delay()) {
-    td::actor::send_closure(session_, &validatorsession::ValidatorSession::set_catchain_max_block_delay,
-                            opts_->get_catchain_max_block_delay().value());
-  }
+  double catchain_delay = opts_->get_catchain_max_block_delay() ? opts_->get_catchain_max_block_delay().value() : 0.4;
+  double catchain_delay_slow =
+      std::max(catchain_delay,
+               opts_->get_catchain_max_block_delay_slow() ? opts_->get_catchain_max_block_delay_slow().value() : 1.0);
+  td::actor::send_closure(session_, &validatorsession::ValidatorSession::set_catchain_max_block_delay, catchain_delay,
+                          catchain_delay_slow);
   if (started_) {
     td::actor::send_closure(session_, &validatorsession::ValidatorSession::start);
   }

validator/validator-options.hpp

@@ -141,6 +141,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   td::optional<double> get_catchain_max_block_delay() const override {
     return catchain_max_block_delay_;
   }
+  td::optional<double> get_catchain_max_block_delay_slow() const override {
+    return catchain_max_block_delay_slow_;
+  }
   bool get_state_serializer_enabled() const override {
     return state_serializer_enabled_;
   }
@@ -230,6 +233,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   void set_catchain_max_block_delay(double value) override {
     catchain_max_block_delay_ = value;
   }
+  void set_catchain_max_block_delay_slow(double value) override {
+    catchain_max_block_delay_slow_ = value;
+  }
   void set_state_serializer_enabled(bool value) override {
     state_serializer_enabled_ = value;
   }
@@ -289,7 +295,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   td::optional<td::uint64> celldb_cache_size_;
   bool celldb_direct_io_ = false;
   bool celldb_preload_all_ = false;
-  td::optional<double> catchain_max_block_delay_;
+  td::optional<double> catchain_max_block_delay_, catchain_max_block_delay_slow_;
   bool state_serializer_enabled_ = true;
   td::Ref<CollatorOptions> collator_options_{true};
   bool fast_state_serializer_enabled_ = false;

validator/validator.h

@@ -105,6 +105,7 @@ struct ValidatorManagerOptions : public td::CntObject {
   virtual bool get_celldb_direct_io() const = 0;
   virtual bool get_celldb_preload_all() const = 0;
   virtual td::optional<double> get_catchain_max_block_delay() const = 0;
+  virtual td::optional<double> get_catchain_max_block_delay_slow() const = 0;
   virtual bool get_state_serializer_enabled() const = 0;
   virtual td::Ref<CollatorOptions> get_collator_options() const = 0;
   virtual bool get_fast_state_serializer_enabled() const = 0;
@@ -136,6 +137,7 @@ struct ValidatorManagerOptions : public td::CntObject {
   virtual void set_celldb_direct_io(bool value) = 0;
   virtual void set_celldb_preload_all(bool value) = 0;
   virtual void set_catchain_max_block_delay(double value) = 0;
+  virtual void set_catchain_max_block_delay_slow(double value) = 0;
   virtual void set_state_serializer_enabled(bool value) = 0;
   virtual void set_collator_options(td::Ref<CollatorOptions> value) = 0;
   virtual void set_fast_state_serializer_enabled(bool value) = 0;