Merge branch 'testnet' into accelerator

validator/manager.cpp

@@ -2260,6 +2260,7 @@ void ValidatorManagerImpl::update_shards() {
     }
   }
 
+  bool validating_masterchain = false;
   if (allow_validate_) {
     for (auto &desc : new_shards) {
       auto shard = desc.first;
@@ -2276,6 +2277,9 @@ void ValidatorManagerImpl::update_shards() {
       auto validator_id = get_validator(shard, val_set);
 
       if (!validator_id.is_zero()) {
+        if (shard.is_masterchain()) {
+          validating_masterchain = true;
+        }
         auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts);
 
         if (force_recover) {
@@ -2368,6 +2372,14 @@ void ValidatorManagerImpl::update_shards() {
       td::actor::send_closure(SelfId, &ValidatorManagerImpl::written_destroyed_validator_sessions, std::move(gc));
     });
     td::actor::send_closure(db_, &Db::update_destroyed_validator_sessions, gc_list_, std::move(P));
+
+    if (!serializer_.empty()) {
+      td::actor::send_closure(
+          serializer_, &AsyncStateSerializer::auto_disable_serializer,
+          validating_masterchain &&
+              last_masterchain_state_->get_validator_set(ShardIdFull{masterchainId})->export_vector().size() * 2 <=
+                  last_masterchain_state_->get_total_validator_set(0)->export_vector().size());
+    }
   }
 }
 

validator/state-serializer.cpp

@@ -94,7 +94,8 @@ void AsyncStateSerializer::request_previous_state_files() {
 }
 
 void AsyncStateSerializer::got_previous_state_files(std::vector<std::pair<std::string, ShardIdFull>> files) {
-  previous_state_files_ = std::move(files);
+  previous_state_cache_ = std::make_shared<PreviousStateCache>();
+  previous_state_cache_->state_files = std::move(files);
   request_masterchain_state();
 }
 
@@ -165,7 +166,10 @@ void AsyncStateSerializer::next_iteration() {
     }
     if (!have_masterchain_state_ && !opts_->get_state_serializer_enabled()) {
       LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str()
-                 << ": serializer is disabled";
+                 << ": serializer is disabled (by user)";
+    } else if (!have_masterchain_state_ && auto_disabled_) {
+      LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str()
+                 << ": serializer is disabled (automatically)";
     } else if (!have_masterchain_state_ && have_newer_persistent_state(masterchain_handle_->unix_time())) {
       LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str()
                  << ": newer key block with ts=" << last_known_key_block_ts_ << " exists";
@@ -174,7 +178,7 @@ void AsyncStateSerializer::next_iteration() {
         LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str();
         // block next attempts immediately, but send actual request later
         running_ = true;
-        double delay = td::Random::fast(0, 3600);
+        double delay = td::Random::fast(0, 3600 * 6);
         LOG(WARNING) << "serializer delay = " << delay << "s";
         delay_action(
             [SelfId = actor_id(this)]() {
@@ -192,9 +196,7 @@ void AsyncStateSerializer::next_iteration() {
     }
     last_key_block_ts_ = masterchain_handle_->unix_time();
     last_key_block_id_ = masterchain_handle_->id();
-    previous_state_files_ = {};
     previous_state_cache_ = {};
-    previous_state_cur_shards_ = {};
   }
   if (!saved_to_db_) {
     running_ = true;
@@ -281,27 +283,24 @@ class CachedCellDbReader : public vm::CellDbReader {
   td::uint64 cached_reqs_ = 0;
 };
 
-void AsyncStateSerializer::prepare_previous_state_cache(ShardIdFull shard) {
+void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) {
-  if (!opts_->get_fast_state_serializer_enabled()) {
-    return;
-  }
   std::vector<ShardIdFull> prev_shards;
-  for (const auto& [_, prev_shard] : previous_state_files_) {
+  for (const auto& [_, prev_shard] : state_files) {
     if (shard_intersects(shard, prev_shard)) {
       prev_shards.push_back(prev_shard);
     }
   }
-  if (prev_shards == previous_state_cur_shards_) {
+  if (prev_shards == cur_shards) {
     return;
   }
-  previous_state_cur_shards_ = std::move(prev_shards);
+  cur_shards = std::move(prev_shards);
-  previous_state_cache_ = {};
+  cache = {};
-  if (previous_state_cur_shards_.empty()) {
+  if (cur_shards.empty()) {
     return;
   }
   td::Timer timer;
   LOG(WARNING) << "Preloading previous persistent state for shard " << shard.to_str() << " ("
-               << previous_state_cur_shards_.size() << " files)";
+               << cur_shards.size() << " files)";
   std::map<td::Bits256, td::Ref<vm::Cell>> cells;
   std::function<void(td::Ref<vm::Cell>)> dfs = [&](td::Ref<vm::Cell> cell) {
     td::Bits256 hash = cell->get_hash().bits();
@@ -314,7 +313,7 @@ void AsyncStateSerializer::prepare_previous_state_cache(ShardIdFull shard) {
       dfs(cs.prefetch_ref(i));
     }
   };
-  for (const auto& [file, prev_shard] : previous_state_files_) {
+  for (const auto& [file, prev_shard] : state_files) {
     if (!shard_intersects(shard, prev_shard)) {
       continue;
     }
@@ -329,22 +328,20 @@ void AsyncStateSerializer::prepare_previous_state_cache(ShardIdFull shard) {
       LOG(WARNING) << "Deserialize error : " << r_root.move_as_error();
       continue;
     }
-    r_data = {};
+    r_data.clear();
     dfs(r_root.move_as_ok());
   }
   LOG(WARNING) << "Preloaded previous state: " << cells.size() << " cells in " << timer.elapsed() << "s";
-  previous_state_cache_ = std::make_shared<std::map<td::Bits256, td::Ref<vm::Cell>>>(std::move(cells));
+  cache = std::make_shared<std::map<td::Bits256, td::Ref<vm::Cell>>>(std::move(cells));
 }
 
 void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state,
                                                  std::shared_ptr<vm::CellDbReader> cell_db_reader) {
-  if (!opts_->get_state_serializer_enabled()) {
+  if (!opts_->get_state_serializer_enabled() || auto_disabled_) {
     stored_masterchain_state();
     return;
   }
   LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str();
-  prepare_previous_state_cache(state->get_shard());
-  auto new_cell_db_reader = std::make_shared<CachedCellDbReader>(cell_db_reader, previous_state_cache_);
   have_masterchain_state_ = true;
   CHECK(next_idx_ == 0);
   CHECK(shards_.size() == 0);
@@ -356,10 +353,16 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state
     }
   }
 
-  auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = new_cell_db_reader,
+  auto write_data = [shard = state->get_shard(), hash = state->root_cell()->get_hash(), cell_db_reader,
+                     previous_state_cache = previous_state_cache_,
+                     fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(),
                      cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable {
-    auto res = vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token));
+    if (fast_serializer_enabled) {
-    cell_db_reader->print_stats();
+      previous_state_cache->prepare_cache(shard);
+    }
+    auto new_cell_db_reader = std::make_shared<CachedCellDbReader>(cell_db_reader, previous_state_cache->cache);
+    auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, hash, fd, 31, std::move(cancellation_token));
+    new_cell_db_reader->print_stats();
     return res;
   };
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
@@ -406,17 +409,21 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) {
 
 void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardState> state,
                                            std::shared_ptr<vm::CellDbReader> cell_db_reader) {
-  if (!opts_->get_state_serializer_enabled()) {
+  if (!opts_->get_state_serializer_enabled() || auto_disabled_) {
     success_handler();
     return;
   }
   LOG(ERROR) << "serializing shard state " << handle->id().id.to_str();
-  prepare_previous_state_cache(state->get_shard());
+  auto write_data = [shard = state->get_shard(), hash = state->root_cell()->get_hash(), cell_db_reader,
-  auto new_cell_db_reader = std::make_shared<CachedCellDbReader>(cell_db_reader, previous_state_cache_);
+                     previous_state_cache = previous_state_cache_,
-  auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = new_cell_db_reader,
+                     fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(),
                      cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable {
-    auto res = vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token));
+    if (fast_serializer_enabled) {
-    cell_db_reader->print_stats();
+      previous_state_cache->prepare_cache(shard);
+    }
+    auto new_cell_db_reader = std::make_shared<CachedCellDbReader>(cell_db_reader, previous_state_cache->cache);
+    auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, hash, fd, 31, std::move(cancellation_token));
+    new_cell_db_reader->print_stats();
     return res;
   };
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Unit> R) {
@@ -458,6 +465,13 @@ void AsyncStateSerializer::update_options(td::Ref<ValidatorManagerOptions> opts)
   }
 }
 
+void AsyncStateSerializer::auto_disable_serializer(bool disabled) {
+  auto_disabled_ = disabled;
+  if (auto_disabled_) {
+    cancellation_token_source_.cancel();
+  }
+}
+
 bool AsyncStateSerializer::need_serialize(BlockHandle handle) {
   if (handle->id().id.seqno == 0 || !handle->is_key_block()) {
     return false;

validator/state-serializer.hpp

@@ -37,6 +37,7 @@ class AsyncStateSerializer : public td::actor::Actor {
   bool saved_to_db_ = true;
 
   td::Ref<ValidatorManagerOptions> opts_;
+  bool auto_disabled_ = false;
   td::CancellationTokenSource cancellation_token_source_;
   UnixTime last_known_key_block_ts_ = 0;
 
@@ -49,11 +50,14 @@ class AsyncStateSerializer : public td::actor::Actor {
   bool have_masterchain_state_ = false;
 
   std::vector<BlockIdExt> shards_;
-  std::vector<std::pair<std::string, ShardIdFull>> previous_state_files_;
+  struct PreviousStateCache {
-  std::shared_ptr<std::map<td::Bits256, td::Ref<vm::Cell>>> previous_state_cache_;
+    std::vector<std::pair<std::string, ShardIdFull>> state_files;
-  std::vector<ShardIdFull> previous_state_cur_shards_;
+    std::shared_ptr<std::map<td::Bits256, td::Ref<vm::Cell>>> cache;
+    std::vector<ShardIdFull> cur_shards;
 
-  void prepare_previous_state_cache(ShardIdFull shard);
+    void prepare_cache(ShardIdFull shard);
+  };
+  std::shared_ptr<PreviousStateCache> previous_state_cache_;
 
  public:
   AsyncStateSerializer(BlockIdExt block_id, td::Ref<ValidatorManagerOptions> opts,
@@ -106,6 +110,7 @@ class AsyncStateSerializer : public td::actor::Actor {
   void success_handler();
 
   void update_options(td::Ref<ValidatorManagerOptions> opts);
+  void auto_disable_serializer(bool disabled);
 };
 
 }  // namespace validator

validator/validator.h

@@ -160,7 +160,7 @@ struct ValidatorManagerOptions : public td::CntObject {
 
       std::function<bool(ShardIdFull)> check_shard = [](ShardIdFull) { return true; },
       bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400,
-      double state_ttl = 3600, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650,
+      double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650,
       double max_mempool_num = 999999, bool initial_sync_disabled = false);
 };