Merge branch 'testnet' into accelerator

crypto/vm/cells/DataCell.cpp

@@ -113,6 +113,9 @@ td::Result<Ref<DataCell>> DataCell::create(td::ConstBitPtr data, unsigned bits,
       if (bits != 8 + hash_bytes * 8) {
         return td::Status::Error("Not enouch data for a Library special cell");
       }
+      if (!refs.empty()) {
+        return td::Status::Error("Library special cell has a cell reference");
+      }
       break;
     }
 

tdutils/td/utils/filesystem.cpp

@@ -68,9 +68,14 @@ Result<T> read_file_impl(CSlice path, int64 size, int64 offset) {
     return Status::Error("Failed to read file: invalid size");
   }
   auto content = create_empty<T>(narrow_cast<size_t>(size));
-  TRY_RESULT(got_size, from_file.pread(as_mutable_slice(content), offset));
+  MutableSlice slice = as_mutable_slice(content);
-  if (got_size != static_cast<size_t>(size)) {
+  while (!slice.empty()) {
-    return Status::Error("Failed to read file");
+    TRY_RESULT(got_size, from_file.pread(slice, offset));
+    if (got_size == 0) {
+      return Status::Error("Failed to read file");
+    }
+    offset += got_size;
+    slice.remove_prefix(got_size);
   }
   from_file.close();
   return std::move(content);
@@ -103,9 +108,15 @@ Status write_file(CSlice to, Slice data, WriteFileOptions options) {
     TRY_STATUS(to_file.lock(FileFd::LockFlags::Write, to.str(), 10));
     TRY_STATUS(to_file.truncate_to_current_position(0));
   }
-  TRY_RESULT(written, to_file.write(data));
+  size_t total_written = 0;
-  if (written != size) {
+  while (!data.empty()) {
-    return Status::Error(PSLICE() << "Failed to write file: written " << written << " bytes instead of " << size);
+    TRY_RESULT(written, to_file.write(data));
+    if (written == 0) {
+      return Status::Error(PSLICE() << "Failed to write file: written " << total_written << " bytes instead of "
+                                    << size);
+    }
+    total_written += written;
+    data.remove_prefix(written);
   }
   if (options.need_sync) {
     TRY_STATUS(to_file.sync());

tl/generate/scheme/ton_api.tl

@@ -775,6 +775,7 @@ engine.validator.showCustomOverlays = engine.validator.CustomOverlaysConfig;
 engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Success;
 
 engine.validator.setCollatorOptionsJson json:string = engine.validator.Success;
+engine.validator.getCollatorOptionsJson = engine.validator.JsonConfig;
 
 engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo;
 

validator-engine-console/validator-engine-console-query.cpp

@@ -1254,6 +1254,27 @@ td::Status ResetCollatorOptionsQuery::receive(td::BufferSlice data) {
   return td::Status::OK();
 }
 
+td::Status GetCollatorOptionsJsonQuery::run() {
+  TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token<std::string>());
+  TRY_STATUS(tokenizer_.check_endl());
+  return td::Status::OK();
+}
+
+td::Status GetCollatorOptionsJsonQuery::send() {
+  auto b =
+      ton::create_serialize_tl_object<ton::ton_api::engine_validator_getCollatorOptionsJson>();
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status GetCollatorOptionsJsonQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_jsonConfig>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  TRY_STATUS(td::write_file(file_name_, f->data_));
+  td::TerminalIO::out() << "saved config to " << file_name_ << "\n";
+  return td::Status::OK();
+}
+
 td::Status GetValidatorSessionsInfoQuery::run() {
   TRY_STATUS(tokenizer_.check_endl());
   return td::Status::OK();

validator-engine-console/validator-engine-console-query.h

@@ -1271,6 +1271,28 @@ class ResetCollatorOptionsQuery : public Query {
   }
 };
 
+class GetCollatorOptionsJsonQuery : public Query {
+ public:
+  GetCollatorOptionsJsonQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "getcollatoroptionsjson";
+  }
+  static std::string get_help() {
+    return "getcollatoroptionsjson <filename>\tsave current collator options to file <filename>";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+
+ private:
+  std::string file_name_;
+};
+
 class GetValidatorSessionsInfoQuery : public Query {
  public:
   GetValidatorSessionsInfoQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)

validator-engine-console/validator-engine-console.cpp

@@ -149,6 +149,7 @@ void ValidatorEngineConsole::run() {
   add_query_runner(std::make_unique<QueryRunnerImpl<SetStateSerializerEnabledQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<SetCollatorOptionsJsonQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<ResetCollatorOptionsQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<GetCollatorOptionsJsonQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<GetValidatorSessionsInfoQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<AddCollatorQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<AddShardQuery>>());

validator-engine/validator-engine.cpp

@@ -3926,6 +3926,26 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollat
   promise.set_value(ton::create_serialize_tl_object<ton::ton_api::engine_validator_success>());
 }
 
+void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query,
+                                        td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
+                                        td::Promise<td::BufferSlice> promise) {
+  if (!(perm & ValidatorEnginePermissions::vep_default)) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
+    return;
+  }
+  if (!started_) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started")));
+    return;
+  }
+  auto r_data = td::read_file(collator_options_file());
+  if (r_data.is_error()) {
+    promise.set_value(ton::create_serialize_tl_object<ton::ton_api::engine_validator_jsonConfig>("{}"));
+  } else {
+    promise.set_value(
+        ton::create_serialize_tl_object<ton::ton_api::engine_validator_jsonConfig>(r_data.ok().as_slice().str()));
+  }
+}
+
 void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollatorsList &query, td::BufferSlice data,
                                         ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise) {
   if (!(perm & ValidatorEnginePermissions::vep_modify)) {

validator-engine/validator-engine.hpp

@@ -530,6 +530,8 @@ class ValidatorEngine : public td::actor::Actor {
                          ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
   void run_control_query(ton::ton_api::engine_validator_importFastSyncMemberCertificate &query, td::BufferSlice data,
                          ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
+  void run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query, td::BufferSlice data,
+                         ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
   template <class T>
   void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
                          td::Promise<td::BufferSlice> promise) {

validator/db/archive-manager.cpp

@@ -310,14 +310,17 @@ void ArchiveManager::get_file(ConstBlockHandle handle, FileReference ref_id, td:
   get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise));
 }
 
-void ArchiveManager::written_perm_state(FileReferenceShort id) {
+void ArchiveManager::register_perm_state(FileReferenceShort id) {
-  perm_states_.emplace(id.hash(), id);
+  BlockSeqno masterchain_seqno = 0;
+  id.ref().visit(td::overloaded(
+      [&](const fileref::PersistentStateShort &x) { masterchain_seqno = x.masterchain_seqno; }, [&](const auto &) {}));
+  perm_states_[{masterchain_seqno, id.hash()}] = id;
 }
 
 void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise<td::Unit> promise) {
   auto id = FileReference{fileref::ZeroState{block_id}};
   auto hash = id.hash();
-  if (perm_states_.find(hash) != perm_states_.end()) {
+  if (perm_states_.find({0, hash}) != perm_states_.end()) {
     promise.set_value(td::Unit());
     return;
   }
@@ -328,7 +331,7 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t
         if (R.is_error()) {
           promise.set_error(R.move_as_error());
         } else {
-          td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id);
+          td::actor::send_closure(SelfId, &ArchiveManager::register_perm_state, id);
           promise.set_value(td::Unit());
         }
       });
@@ -357,12 +360,13 @@ void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt ma
   add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer));
 }
 
-void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+void ArchiveManager::add_persistent_state_impl(
-                                               td::Promise<td::Unit> promise,
+    BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise,
-                                               std::function<void(std::string, td::Promise<std::string>)> create_writer) {
+    std::function<void(std::string, td::Promise<std::string>)> create_writer) {
   auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
+  BlockSeqno masterchain_seqno = masterchain_block_id.seqno();
   auto hash = id.hash();
-  if (perm_states_.find(hash) != perm_states_.end()) {
+  if (perm_states_.find({masterchain_seqno, hash}) != perm_states_.end()) {
     promise.set_value(td::Unit());
     return;
   }
@@ -373,7 +377,7 @@ void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt m
         if (R.is_error()) {
           promise.set_error(R.move_as_error());
         } else {
-          td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id);
+          td::actor::send_closure(SelfId, &ArchiveManager::register_perm_state, id);
           promise.set_value(td::Unit());
         }
       });
@@ -383,7 +387,7 @@ void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt m
 void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {
   auto id = FileReference{fileref::ZeroState{block_id}};
   auto hash = id.hash();
-  if (perm_states_.find(hash) == perm_states_.end()) {
+  if (perm_states_.find({0, hash}) == perm_states_.end()) {
     promise.set_error(td::Status::Error(ErrorCode::notready, "zerostate not in db"));
     return;
   }
@@ -395,18 +399,38 @@ void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise<td::BufferS
 void ArchiveManager::check_zero_state(BlockIdExt block_id, td::Promise<bool> promise) {
   auto id = FileReference{fileref::ZeroState{block_id}};
   auto hash = id.hash();
-  if (perm_states_.find(hash) == perm_states_.end()) {
+  if (perm_states_.find({0, hash}) == perm_states_.end()) {
     promise.set_result(false);
     return;
   }
   promise.set_result(true);
 }
 
+void ArchiveManager::get_previous_persistent_state_files(
+    BlockSeqno cur_mc_seqno, td::Promise<std::vector<std::pair<std::string, ShardIdFull>>> promise) {
+  auto it = perm_states_.lower_bound({cur_mc_seqno, FileHash::zero()});
+  if (it == perm_states_.begin()) {
+    promise.set_value({});
+    return;
+  }
+  --it;
+  BlockSeqno mc_seqno = it->first.first;
+  std::vector<std::pair<std::string, ShardIdFull>> files;
+  while (it->first.first == mc_seqno) {
+    files.emplace_back(db_root_ + "/archive/states/" + it->second.filename_short(), it->second.shard());
+    if (it == perm_states_.begin()) {
+      break;
+    }
+    --it;
+  }
+  promise.set_value(std::move(files));
+}
+
 void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id,
                                           td::Promise<td::BufferSlice> promise) {
   auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
   auto hash = id.hash();
-  if (perm_states_.find(hash) == perm_states_.end()) {
+  if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) {
     promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db"));
     return;
   }
@@ -419,7 +443,7 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt
                                                 td::int64 max_size, td::Promise<td::BufferSlice> promise) {
   auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
   auto hash = id.hash();
-  if (perm_states_.find(hash) == perm_states_.end()) {
+  if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) {
     promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db"));
     return;
   }
@@ -432,7 +456,7 @@ void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt mast
                                             td::Promise<bool> promise) {
   auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
   auto hash = id.hash();
-  if (perm_states_.find(hash) == perm_states_.end()) {
+  if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) {
     promise.set_result(false);
     return;
   }
@@ -885,13 +909,11 @@ void ArchiveManager::start_up() {
         R = FileReferenceShort::create(newfname);
         R.ensure();
       }
-      auto f = R.move_as_ok();
+      register_perm_state(R.move_as_ok());
-      auto hash = f.hash();
-      perm_states_[hash] = std::move(f);
     }
   }).ensure();
 
-  persistent_state_gc(FileHash::zero());
+  persistent_state_gc({0, FileHash::zero()});
 
   double open_since = td::Clocks::system() - opts_->get_archive_preload_period();
   for (auto it = files_.rbegin(); it != files_.rend(); ++it) {
@@ -977,11 +999,12 @@ void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl
   }
 }
 
-void ArchiveManager::persistent_state_gc(FileHash last) {
+void ArchiveManager::persistent_state_gc(std::pair<BlockSeqno, FileHash> last) {
-  if (perm_states_.size() == 0) {
+  if (perm_states_.empty()) {
     delay_action(
-        [hash = FileHash::zero(), SelfId = actor_id(this)]() {
+        [SelfId = actor_id(this)]() {
-          td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash);
+          td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc,
+                                  std::pair<BlockSeqno, FileHash>{0, FileHash::zero()});
         },
         td::Timestamp::in(1.0));
     return;
@@ -994,12 +1017,12 @@ void ArchiveManager::persistent_state_gc(FileHash last) {
     it = perm_states_.begin();
   }
 
+  auto key = it->first;
   auto &F = it->second;
-  auto hash = F.hash();
 
   int res = 0;
   BlockSeqno seqno = 0;
-  F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &x) { res = 1; },
+  F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; },
                                [&](const fileref::PersistentStateShort &x) {
                                  res = 0;
                                  seqno = x.masterchain_seqno;
@@ -1011,24 +1034,41 @@ void ArchiveManager::persistent_state_gc(FileHash last) {
     perm_states_.erase(it);
   }
   if (res != 0) {
-    delay_action([hash, SelfId = actor_id(
+    delay_action([key, SelfId = actor_id(
-                            this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); },
+                            this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
+                 td::Timestamp::in(1.0));
+    return;
+  }
+  CHECK(seqno == key.first);
+
+  // Do not delete the most recent fully serialized state
+  bool allow_delete = false;
+  auto it2 = perm_states_.lower_bound({seqno + 1, FileHash::zero()});
+  if (it2 != perm_states_.end()) {
+    it2 = perm_states_.lower_bound({it2->first.first + 1, FileHash::zero()});
+    if (it2 != perm_states_.end()) {
+      allow_delete = true;
+    }
+  }
+  if (!allow_delete) {
+    delay_action([key, SelfId = actor_id(
+                            this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
                  td::Timestamp::in(1.0));
     return;
   }
 
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), hash](td::Result<ConstBlockHandle> R) {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), key](td::Result<ConstBlockHandle> R) {
     if (R.is_error()) {
-      td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, nullptr, hash);
+      td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, nullptr, key);
     } else {
-      td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, R.move_as_ok(), hash);
+      td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, R.move_as_ok(), key);
     }
   });
 
   get_block_by_seqno(AccountIdPrefixFull{masterchainId, 0}, seqno, std::move(P));
 }
 
-void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, FileHash hash) {
+void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pair<BlockSeqno, FileHash> key) {
   bool to_del = false;
   if (!handle || !handle->inited_unix_time() || !handle->unix_time()) {
     to_del = true;
@@ -1036,15 +1076,15 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, FileHash
     auto ttl = ValidatorManager::persistent_state_ttl(handle->unix_time());
     to_del = ttl < td::Clocks::system();
   }
-  auto it = perm_states_.find(hash);
+  auto it = perm_states_.find(key);
   CHECK(it != perm_states_.end());
   auto &F = it->second;
   if (to_del) {
     td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore();
     perm_states_.erase(it);
   }
-  delay_action([hash, SelfId = actor_id(
+  delay_action([key, SelfId = actor_id(
-                          this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); },
+                          this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
                td::Timestamp::in(1.0));
 }
 

validator/db/archive-manager.hpp

@@ -54,6 +54,8 @@ class ArchiveManager : public td::actor::Actor {
                                   td::int64 max_size, td::Promise<td::BufferSlice> promise);
   void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<bool> promise);
   void check_zero_state(BlockIdExt block_id, td::Promise<bool> promise);
+  void get_previous_persistent_state_files(BlockSeqno cur_mc_seqno,
+                                           td::Promise<std::vector<std::pair<std::string, ShardIdFull>>> promise);
 
   void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise);
   //void truncate_continue(BlockSeqno masterchain_seqno, td::Promise<td::Unit> promise);
@@ -185,7 +187,7 @@ class ArchiveManager : public td::actor::Actor {
     return p.key ? key_files_ : p.temp ? temp_files_ : files_;
   }
 
-  std::map<FileHash, FileReferenceShort> perm_states_;
+  std::map<std::pair<BlockSeqno, FileHash>, FileReferenceShort> perm_states_;  // Mc block seqno, hash -> state
 
   void load_package(PackageId seqno);
   void delete_package(PackageId seqno, td::Promise<td::Unit> promise);
@@ -212,10 +214,10 @@ class ArchiveManager : public td::actor::Actor {
 
   void add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise,
                                  std::function<void(std::string, td::Promise<std::string>)> create_writer);
-  void written_perm_state(FileReferenceShort id);
+  void register_perm_state(FileReferenceShort id);
 
-  void persistent_state_gc(FileHash last);
+  void persistent_state_gc(std::pair<BlockSeqno, FileHash> last);
-  void got_gc_masterchain_handle(ConstBlockHandle handle, FileHash hash);
+  void got_gc_masterchain_handle(ConstBlockHandle handle, std::pair<BlockSeqno, FileHash> key);
 
   std::string db_root_;
   td::Ref<ValidatorManagerOptions> opts_;

validator/db/rootdb.cpp

@@ -317,6 +317,12 @@ void RootDb::check_zero_state_file_exists(BlockIdExt block_id, td::Promise<bool>
   td::actor::send_closure(archive_db_, &ArchiveManager::check_zero_state, block_id, std::move(promise));
 }
 
+void RootDb::get_previous_persistent_state_files(
+    BlockSeqno cur_mc_seqno, td::Promise<std::vector<std::pair<std::string, ShardIdFull>>> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_previous_persistent_state_files, cur_mc_seqno,
+                          std::move(promise));
+}
+
 void RootDb::store_block_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
   td::actor::send_closure(archive_db_, &ArchiveManager::update_handle, std::move(handle), std::move(promise));
 }

validator/db/rootdb.hpp

@@ -84,6 +84,8 @@ class RootDb : public Db {
   void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise<td::Unit> promise) override;
   void get_zero_state_file(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) override;
   void check_zero_state_file_exists(BlockIdExt block_id, td::Promise<bool> promise) override;
+  void get_previous_persistent_state_files(
+      BlockSeqno cur_mc_seqno, td::Promise<std::vector<std::pair<std::string, ShardIdFull>>> promise) override;
 
   void try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice> promise) override;
 

validator/full-node-shard.cpp

@@ -950,8 +950,7 @@ void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt mast
   auto &b = choose_neighbour();
   td::actor::create_actor<DownloadState>(PSTRING() << "downloadstatereq" << id.id.to_str(), id, masterchain_block_id,
                                          adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_,
-                                         b.use_rldp2() ? (td::actor::ActorId<adnl::AdnlSenderInterface>)rldp2_ : rldp_,
+                                         rldp2_, overlays_, adnl_, client_, std::move(promise))
-                                         overlays_, adnl_, client_, std::move(promise))
       .release();
 }
 
@@ -987,8 +986,7 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFu
   auto &b = choose_neighbour(true);
   td::actor::create_actor<DownloadArchiveSlice>(
       "archive", masterchain_seqno, shard_prefix, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout,
-      validator_manager_, b.use_rldp2() ? (td::actor::ActorId<adnl::AdnlSenderInterface>)rldp2_ : rldp_, overlays_,
+      validator_manager_, rldp2_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise)))
-      adnl_, client_, create_neighbour_promise(b, std::move(promise)))
       .release();
 }
 

validator/full-node-shard.hpp

@@ -47,9 +47,6 @@ struct Neighbour {
   void query_failed();
   void update_roundtrip(double t);
 
-  bool use_rldp2() const {
-    return std::make_pair(version_major, version_minor) >= std::make_pair<td::uint32, td::uint32>(2, 2);
-  }
   bool has_state() const {
     return !(flags & FLAG_NO_STATE);
   }

validator/interfaces/db.h

@@ -66,6 +66,8 @@ class Db : public td::actor::Actor {
   virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise<td::Unit> promise) = 0;
   virtual void get_zero_state_file(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) = 0;
   virtual void check_zero_state_file_exists(BlockIdExt block_id, td::Promise<bool> promise) = 0;
+  virtual void get_previous_persistent_state_files(
+      BlockSeqno cur_mc_seqno, td::Promise<std::vector<std::pair<std::string, ShardIdFull>>> promise) = 0;
 
   virtual void try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice> promise) = 0;
 

validator/manager-disk.hpp

@@ -116,6 +116,10 @@ class ValidatorManagerImpl : public ValidatorManager {
                                   td::int64 max_length, td::Promise<td::BufferSlice> promise) override {
     UNREACHABLE();
   }
+  void get_previous_persistent_state_files(
+      BlockSeqno cur_mc_seqno, td::Promise<std::vector<std::pair<std::string, ShardIdFull>>> promise) override {
+    UNREACHABLE();
+  }
   void get_block_proof(BlockHandle handle, td::Promise<td::BufferSlice> promise) override;
   void get_block_proof_link(BlockHandle block_id, td::Promise<td::BufferSlice> promise) override {
     UNREACHABLE();

validator/manager-hardfork.hpp

@@ -140,6 +140,10 @@ class ValidatorManagerImpl : public ValidatorManager {
                                   td::int64 max_length, td::Promise<td::BufferSlice> promise) override {
     UNREACHABLE();
   }
+  void get_previous_persistent_state_files(
+      BlockSeqno cur_mc_seqno, td::Promise<std::vector<std::pair<std::string, ShardIdFull>>> promise) override {
+    UNREACHABLE();
+  }
   void get_block_proof(BlockHandle handle, td::Promise<td::BufferSlice> promise) override;
   void get_block_proof_link(BlockHandle block_id, td::Promise<td::BufferSlice> promise) override;
   void get_key_block_proof(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) override;

validator/manager.cpp

@@ -326,6 +326,11 @@ void ValidatorManagerImpl::get_persistent_state_slice(BlockIdExt block_id, Block
                           std::move(promise));
 }
 
+void ValidatorManagerImpl::get_previous_persistent_state_files(
+    BlockSeqno cur_mc_seqno, td::Promise<std::vector<std::pair<std::string, ShardIdFull>>> promise) {
+  td::actor::send_closure(db_, &Db::get_previous_persistent_state_files, cur_mc_seqno, std::move(promise));
+}
+
 void ValidatorManagerImpl::get_block_proof(BlockHandle handle, td::Promise<td::BufferSlice> promise) {
   auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Ref<Proof>> R) mutable {
     if (R.is_error()) {
@@ -1894,6 +1899,8 @@ void ValidatorManagerImpl::read_gc_list(std::vector<ValidatorSessionId> list) {
 
   serializer_ =
       td::actor::create_actor<AsyncStateSerializer>("serializer", last_key_block_handle_->id(), opts_, actor_id(this));
+  td::actor::send_closure(serializer_, &AsyncStateSerializer::update_last_known_key_block_ts,
+                          last_key_block_handle_->unix_time());
 
   if (last_masterchain_block_handle_->inited_next_left()) {
     auto b = last_masterchain_block_handle_->one_next(true);
@@ -2057,6 +2064,8 @@ void ValidatorManagerImpl::new_masterchain_block() {
       last_known_key_block_handle_ = last_key_block_handle_;
       callback_->new_key_block(last_key_block_handle_);
     }
+    td::actor::send_closure(serializer_, &AsyncStateSerializer::update_last_known_key_block_ts,
+                            last_key_block_handle_->unix_time());
   }
 
   update_shard_overlays();

validator/manager.hpp

@@ -389,6 +389,8 @@ class ValidatorManagerImpl : public ValidatorManager {
                             td::Promise<td::BufferSlice> promise) override;
   void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,
                                   td::int64 max_length, td::Promise<td::BufferSlice> promise) override;
+  void get_previous_persistent_state_files(
+      BlockSeqno cur_mc_seqno, td::Promise<std::vector<std::pair<std::string, ShardIdFull>>> promise) override;
   void get_block_proof(BlockHandle handle, td::Promise<td::BufferSlice> promise) override;
   void get_block_proof_link(BlockHandle block_id, td::Promise<td::BufferSlice> promise) override;
   void get_key_block_proof(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) override;

validator/state-serializer.cpp

@@ -20,6 +20,7 @@
 #include "td/utils/Random.h"
 #include "ton/ton-io.hpp"
 #include "common/delay.h"
+#include "td/utils/filesystem.h"
 
 namespace ton {
 
@@ -83,6 +84,20 @@ void AsyncStateSerializer::alarm() {
   td::actor::send_closure(manager_, &ValidatorManager::get_top_masterchain_block, std::move(P));
 }
 
+void AsyncStateSerializer::request_previous_state_files() {
+  td::actor::send_closure(
+      manager_, &ValidatorManager::get_previous_persistent_state_files, masterchain_handle_->id().seqno(),
+      [SelfId = actor_id(this)](td::Result<std::vector<std::pair<std::string, ShardIdFull>>> R) {
+        R.ensure();
+        td::actor::send_closure(SelfId, &AsyncStateSerializer::got_previous_state_files, R.move_as_ok());
+      });
+}
+
+void AsyncStateSerializer::got_previous_state_files(std::vector<std::pair<std::string, ShardIdFull>> files) {
+  previous_state_files_ = std::move(files);
+  request_masterchain_state();
+}
+
 void AsyncStateSerializer::request_masterchain_state() {
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), manager = manager_](td::Result<td::Ref<ShardState>> R) {
     if (R.is_error()) {
@@ -132,7 +147,7 @@ void AsyncStateSerializer::next_iteration() {
   }
   CHECK(masterchain_handle_->id() == last_block_id_);
   if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno &&
-      need_serialize(masterchain_handle_) && opts_->get_state_serializer_enabled()) {
+      need_serialize(masterchain_handle_)) {
     if (!stored_persistent_state_description_) {
       LOG(INFO) << "storing persistent state description for " << masterchain_handle_->id().id;
       running_ = true;
@@ -148,32 +163,38 @@ void AsyncStateSerializer::next_iteration() {
       td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P));
       return;
     }
-    if (!have_masterchain_state_) {
+    if (!have_masterchain_state_ && !opts_->get_state_serializer_enabled()) {
-      LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str();
+      LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str()
-      // block next attempts immediately, but send actual request later
+                 << ": serializer is disabled";
-      running_ = true;
+    } else if (!have_masterchain_state_ && have_newer_persistent_state(masterchain_handle_->unix_time())) {
-      double delay = td::Random::fast(0, 3600);
+      LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str()
-      LOG(WARNING) << "serializer delay = " << delay << "s";
+                 << ": newer key block with ts=" << last_known_key_block_ts_ << " exists";
-      delay_action([SelfId = actor_id(
+    } else {
-                        this)]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_masterchain_state); },
+      if (!have_masterchain_state_) {
-                   td::Timestamp::in(delay));
+        LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str();
-      return;
+        // block next attempts immediately, but send actual request later
+        running_ = true;
+        double delay = td::Random::fast(0, 3600);
+        LOG(WARNING) << "serializer delay = " << delay << "s";
+        delay_action(
+            [SelfId = actor_id(this)]() {
+              td::actor::send_closure(SelfId, &AsyncStateSerializer::request_previous_state_files);
+            },
+            td::Timestamp::in(delay));
+        return;
+      }
+      if (next_idx_ < shards_.size()) {
+        running_ = true;
+        request_shard_state(shards_[next_idx_]);
+        return;
+      }
+      LOG(ERROR) << "finished serializing persistent state for " << masterchain_handle_->id().id.to_str();
     }
-    while (next_idx_ < shards_.size()) {
-      // block next attempts immediately, but send actual request later
-      running_ = true;
-      double delay = td::Random::fast(0, 1800);
-      LOG(WARNING) << "serializer delay = " << delay << "s";
-      delay_action(
-          [SelfId = actor_id(this), shard = shards_[next_idx_]]() {
-            td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard);
-          },
-          td::Timestamp::in(delay));
-      return;
-    }
-    LOG(ERROR) << "finished serializing persistent state for " << masterchain_handle_->id().id.to_str();
     last_key_block_ts_ = masterchain_handle_->unix_time();
     last_key_block_id_ = masterchain_handle_->id();
+    previous_state_files_ = {};
+    previous_state_cache_ = {};
+    previous_state_cur_shards_ = {};
   }
   if (!saved_to_db_) {
     running_ = true;
@@ -187,9 +208,6 @@ void AsyncStateSerializer::next_iteration() {
     return;
   }
   if (masterchain_handle_->inited_next_left()) {
-    if (need_serialize(masterchain_handle_) && !opts_->get_state_serializer_enabled()) {
-      LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str();
-    }
     last_block_id_ = masterchain_handle_->one_next(true);
     have_masterchain_state_ = false;
     stored_persistent_state_description_ = false;
@@ -233,6 +251,88 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) {
   next_iteration();
 }
 
+class CachedCellDbReader : public vm::CellDbReader {
+ public:
+  CachedCellDbReader(std::shared_ptr<vm::CellDbReader> parent,
+                     std::shared_ptr<std::map<td::Bits256, td::Ref<vm::Cell>>> cache)
+      : parent_(std::move(parent)), cache_(std::move(cache)) {
+  }
+  td::Result<td::Ref<vm::DataCell>> load_cell(td::Slice hash) override {
+    ++total_reqs_;
+    DCHECK(hash.size() == 32);
+    if (cache_) {
+      auto it = cache_->find(td::Bits256{(const unsigned char*)hash.data()});
+      if (it != cache_->end()) {
+        ++cached_reqs_;
+        TRY_RESULT(loaded_cell, it->second->load_cell());
+        return loaded_cell.data_cell;
+      }
+    }
+    return parent_->load_cell(hash);
+  }
+  void print_stats() const {
+    LOG(WARNING) << "CachedCellDbReader stats : " << total_reqs_ << " reads, " << cached_reqs_ << " cached";
+  }
+ private:
+  std::shared_ptr<vm::CellDbReader> parent_;
+  std::shared_ptr<std::map<td::Bits256, td::Ref<vm::Cell>>> cache_;
+
+  td::uint64 total_reqs_ = 0;
+  td::uint64 cached_reqs_ = 0;
+};
+
+void AsyncStateSerializer::prepare_previous_state_cache(ShardIdFull shard) {
+  std::vector<ShardIdFull> prev_shards;
+  for (const auto& [_, prev_shard] : previous_state_files_) {
+    if (shard_intersects(shard, prev_shard)) {
+      prev_shards.push_back(prev_shard);
+    }
+  }
+  if (prev_shards == previous_state_cur_shards_) {
+    return;
+  }
+  previous_state_cur_shards_ = std::move(prev_shards);
+  previous_state_cache_ = {};
+  if (previous_state_cur_shards_.empty()) {
+    return;
+  }
+  td::Timer timer;
+  LOG(WARNING) << "Preloading previous persistent state for shard " << shard.to_str() << " ("
+               << previous_state_cur_shards_.size() << " files)";
+  std::map<td::Bits256, td::Ref<vm::Cell>> cells;
+  std::function<void(td::Ref<vm::Cell>)> dfs = [&](td::Ref<vm::Cell> cell) {
+    td::Bits256 hash = cell->get_hash().bits();
+    if (!cells.emplace(hash, cell).second) {
+      return;
+    }
+    bool is_special;
+    vm::CellSlice cs = vm::load_cell_slice_special(cell, is_special);
+    for (unsigned i = 0; i < cs.size_refs(); ++i) {
+      dfs(cs.prefetch_ref(i));
+    }
+  };
+  for (const auto& [file, prev_shard] : previous_state_files_) {
+    if (!shard_intersects(shard, prev_shard)) {
+      continue;
+    }
+    auto r_data = td::read_file(file);
+    if (r_data.is_error()) {
+      LOG(INFO) << "Reading " << file << " : " << r_data.move_as_error();
+      continue;
+    }
+    LOG(INFO) << "Reading " << file << " : " << td::format::as_size(r_data.ok().size());
+    auto r_root = vm::std_boc_deserialize(r_data.move_as_ok());
+    if (r_root.is_error()) {
+      LOG(WARNING) << "Deserialize error : " << r_root.move_as_error();
+      continue;
+    }
+    r_data = {};
+    dfs(r_root.move_as_ok());
+  }
+  LOG(WARNING) << "Preloaded previous state: " << cells.size() << " cells in " << timer.elapsed() << "s";
+  previous_state_cache_ = std::make_shared<std::map<td::Bits256, td::Ref<vm::Cell>>>(std::move(cells));
+}
+
 void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state,
                                                  std::shared_ptr<vm::CellDbReader> cell_db_reader) {
   if (!opts_->get_state_serializer_enabled()) {
@@ -240,6 +340,8 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state
     return;
   }
   LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str();
+  prepare_previous_state_cache(state->get_shard());
+  auto new_cell_db_reader = std::make_shared<CachedCellDbReader>(cell_db_reader, previous_state_cache_);
   have_masterchain_state_ = true;
   CHECK(next_idx_ == 0);
   CHECK(shards_.size() == 0);
@@ -251,9 +353,11 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state
     }
   }
 
-  auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader,
+  auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = new_cell_db_reader,
                      cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable {
-    return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token));
+    auto res = vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token));
+    cell_db_reader->print_stats();
+    return res;
   };
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
     if (R.is_error() && R.error().code() == cancelled) {
@@ -304,9 +408,13 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardStat
     return;
   }
   LOG(ERROR) << "serializing shard state " << handle->id().id.to_str();
-  auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader,
+  prepare_previous_state_cache(state->get_shard());
+  auto new_cell_db_reader = std::make_shared<CachedCellDbReader>(cell_db_reader, previous_state_cache_);
+  auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = new_cell_db_reader,
                      cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable {
-    return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token));
+    auto res = vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token));
+    cell_db_reader->print_stats();
+    return res;
   };
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Unit> R) {
     if (R.is_error() && R.error().code() == cancelled) {
@@ -355,6 +463,10 @@ bool AsyncStateSerializer::need_serialize(BlockHandle handle) {
          ValidatorManager::persistent_state_ttl(handle->unix_time()) > (UnixTime)td::Clocks::system();
 }
 
+bool AsyncStateSerializer::have_newer_persistent_state(UnixTime cur_ts) {
+  return cur_ts / (1 << 17) < last_known_key_block_ts_ / (1 << 17);
+}
+
 }  // namespace validator
 
 }  // namespace ton

validator/state-serializer.hpp

@@ -38,6 +38,7 @@ class AsyncStateSerializer : public td::actor::Actor {
 
   td::Ref<ValidatorManagerOptions> opts_;
   td::CancellationTokenSource cancellation_token_source_;
+  UnixTime last_known_key_block_ts_ = 0;
 
   td::actor::ActorId<ValidatorManager> manager_;
 
@@ -48,6 +49,11 @@ class AsyncStateSerializer : public td::actor::Actor {
   bool have_masterchain_state_ = false;
 
   std::vector<BlockIdExt> shards_;
+  std::vector<std::pair<std::string, ShardIdFull>> previous_state_files_;
+  std::shared_ptr<std::map<td::Bits256, td::Ref<vm::Cell>>> previous_state_cache_;
+  std::vector<ShardIdFull> previous_state_cur_shards_;
+
+  void prepare_previous_state_cache(ShardIdFull shard);
 
  public:
   AsyncStateSerializer(BlockIdExt block_id, td::Ref<ValidatorManagerOptions> opts,
@@ -60,12 +66,15 @@ class AsyncStateSerializer : public td::actor::Actor {
   }
 
   bool need_serialize(BlockHandle handle);
+  bool have_newer_persistent_state(UnixTime cur_ts);
 
   void alarm() override;
   void start_up() override;
   void got_self_state(AsyncSerializerState state);
   void got_init_handle(BlockHandle handle);
 
+  void request_previous_state_files();
+  void got_previous_state_files(std::vector<std::pair<std::string, ShardIdFull>> files);
   void request_masterchain_state();
   void request_shard_state(BlockIdExt shard);
 
@@ -82,6 +91,10 @@ class AsyncStateSerializer : public td::actor::Actor {
     promise.set_result(last_block_id_.id.seqno);
   }
 
+  void update_last_known_key_block_ts(UnixTime ts) {
+    last_known_key_block_ts_ = std::max(last_known_key_block_ts_, ts);
+  }
+
   void saved_to_db() {
     saved_to_db_ = true;
     running_ = false;

validator/validator.h

@@ -235,6 +235,8 @@ class ValidatorManagerInterface : public td::actor::Actor {
                                     td::Promise<td::BufferSlice> promise) = 0;
   virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,
                                           td::int64 max_length, td::Promise<td::BufferSlice> promise) = 0;
+  virtual void get_previous_persistent_state_files(
+      BlockSeqno cur_mc_seqno, td::Promise<std::vector<std::pair<std::string, ShardIdFull>>> promise) = 0;
   virtual void get_block_proof(BlockHandle handle, td::Promise<td::BufferSlice> promise) = 0;
   virtual void get_block_proof_link(BlockHandle handle, td::Promise<td::BufferSlice> promise) = 0;
   virtual void get_block_handle(BlockIdExt block_id, bool force, td::Promise<BlockHandle> promise) = 0;