new db

new database
fift/func bugfixes

validator/CMakeLists.txt

@@ -10,17 +10,17 @@ add_subdirectory(impl)
 set(VALIDATOR_DB_SOURCE
   db/archiver.cpp
   db/archiver.hpp
-  db/archive-db.cpp
-  db/archive-db.hpp
+  db/archive-manager.cpp
+  db/archive-manager.hpp
+  db/archive-slice.cpp
+  db/archive-slice.hpp
   db/blockdb.cpp
   db/blockdb.hpp
   db/celldb.cpp
   db/celldb.hpp
   db/files-async.hpp
-  db/filedb.hpp
-  db/filedb.cpp
-  db/ltdb.hpp
-  db/ltdb.cpp
+  db/fileref.hpp
+  db/fileref.cpp
   db/rootdb.cpp
   db/rootdb.hpp
   db/statedb.hpp
@@ -52,6 +52,8 @@ set(VALIDATOR_HEADERS
   interfaces/validator-manager.h
   interfaces/validator-set.h
   invariants.hpp
+  
+  import-db-slice.hpp
 
   manager-disk.h
   manager-disk.hpp
@@ -69,6 +71,7 @@ set(VALIDATOR_SOURCE
   apply-block.cpp
   block-handle.cpp
   get-next-key-blocks.cpp
+  import-db-slice.cpp
   shard-client.cpp
   state-serializer.cpp
   token-manager.cpp
@@ -124,6 +127,8 @@ set(FULL_NODE_SOURCE
   net/download-block.cpp
   net/download-block-new.hpp
   net/download-block-new.cpp
+  net/download-archive-slice.hpp
+  net/download-archive-slice.cpp
   net/download-next-block.hpp
   net/download-next-block.cpp
   net/download-state.hpp

validator/apply-block.cpp

@@ -53,6 +53,10 @@ void ApplyBlock::alarm() {
 void ApplyBlock::start_up() {
   VLOG(VALIDATOR_DEBUG) << "running apply_block for " << id_;
 
+  if (id_.is_masterchain()) {
+    masterchain_block_id_ = id_;
+  }
+
   alarm_timestamp() = timeout_;
 
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
@@ -144,6 +148,7 @@ void ApplyBlock::got_block_handle(BlockHandle handle) {
 }
 
 void ApplyBlock::written_block_data() {
+  VLOG(VALIDATOR_DEBUG) << "apply block: written block data for " << id_;
   if (handle_->is_applied() && handle_->processed()) {
     finish_query();
   } else {
@@ -161,6 +166,7 @@ void ApplyBlock::written_block_data() {
 }
 
 void ApplyBlock::got_cur_state(td::Ref<ShardState> state) {
+  VLOG(VALIDATOR_DEBUG) << "apply block: received state for " << id_;
   state_ = std::move(state);
   CHECK(handle_->received_state());
   written_state();
@@ -171,6 +177,7 @@ void ApplyBlock::written_state() {
     finish_query();
     return;
   }
+  VLOG(VALIDATOR_DEBUG) << "apply block: setting next for parents of " << id_;
 
   if (handle_->id().id.seqno != 0 && !handle_->is_applied()) {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
@@ -201,10 +208,12 @@ void ApplyBlock::written_next() {
     return;
   }
 
-  if (handle_->id().id.seqno != 0) {
+  VLOG(VALIDATOR_DEBUG) << "apply block: applying parents of " << id_;
+
+  if (handle_->id().id.seqno != 0 && !handle_->is_applied()) {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
       if (R.is_error()) {
-        td::actor::send_closure(SelfId, &ApplyBlock::abort_query, R.move_as_error());
+        td::actor::send_closure(SelfId, &ApplyBlock::abort_query, R.move_as_error_prefix("prev: "));
       } else {
         td::actor::send_closure(SelfId, &ApplyBlock::applied_prev);
       }
@@ -213,9 +222,13 @@ void ApplyBlock::written_next() {
     td::MultiPromise mp;
     auto g = mp.init_guard();
     g.add_promise(std::move(P));
-    run_apply_block_query(handle_->one_prev(true), td::Ref<BlockData>{}, manager_, timeout_, g.get_promise());
+    BlockIdExt m = masterchain_block_id_;
+    if (id_.is_masterchain()) {
+      m = id_;
+    }
+    run_apply_block_query(handle_->one_prev(true), td::Ref<BlockData>{}, m, manager_, timeout_, g.get_promise());
     if (handle_->merge_before()) {
-      run_apply_block_query(handle_->one_prev(false), td::Ref<BlockData>{}, manager_, timeout_, g.get_promise());
+      run_apply_block_query(handle_->one_prev(false), td::Ref<BlockData>{}, m, manager_, timeout_, g.get_promise());
     }
   } else {
     applied_prev();
@@ -223,6 +236,10 @@ void ApplyBlock::written_next() {
 }
 
 void ApplyBlock::applied_prev() {
+  VLOG(VALIDATOR_DEBUG) << "apply block: waiting manager's confirm for " << id_;
+  if (!id_.is_masterchain()) {
+    handle_->set_masterchain_ref_block(masterchain_block_id_.seqno());
+  }
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
     if (R.is_error()) {
       td::actor::send_closure(SelfId, &ApplyBlock::abort_query, R.move_as_error());
@@ -234,7 +251,12 @@ void ApplyBlock::applied_prev() {
 }
 
 void ApplyBlock::applied_set() {
+  VLOG(VALIDATOR_DEBUG) << "apply block: setting apply bit for " << id_;
   handle_->set_applied();
+  if (handle_->id().seqno() > 0) {
+    CHECK(handle_->handle_moved_to_archive());
+    CHECK(handle_->moved_to_archive());
+  }
   if (handle_->need_flush()) {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
       if (R.is_error()) {

validator/apply-block.hpp

@@ -39,9 +39,14 @@ namespace validator {
 
 class ApplyBlock : public td::actor::Actor {
  public:
-  ApplyBlock(BlockIdExt id, td::Ref<BlockData> block, td::actor::ActorId<ValidatorManager> manager,
-             td::Timestamp timeout, td::Promise<td::Unit> promise)
-      : id_(id), block_(std::move(block)), manager_(manager), timeout_(timeout), promise_(std::move(promise)) {
+  ApplyBlock(BlockIdExt id, td::Ref<BlockData> block, BlockIdExt masterchain_block_id,
+             td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout, td::Promise<td::Unit> promise)
+      : id_(id)
+      , block_(std::move(block))
+      , masterchain_block_id_(masterchain_block_id)
+      , manager_(manager)
+      , timeout_(timeout)
+      , promise_(std::move(promise)) {
   }
 
   static constexpr td::uint32 apply_block_priority() {
@@ -65,6 +70,7 @@ class ApplyBlock : public td::actor::Actor {
  private:
   BlockIdExt id_;
   td::Ref<BlockData> block_;
+  BlockIdExt masterchain_block_id_;
   td::actor::ActorId<ValidatorManager> manager_;
   td::Timestamp timeout_;
   td::Promise<td::Unit> promise_;

validator/block-handle.cpp

@@ -32,18 +32,19 @@ void BlockHandleImpl::flush(td::actor::ActorId<ValidatorManagerInterface> manage
 td::BufferSlice BlockHandleImpl::serialize() const {
   while (locked()) {
   }
-  auto flags = flags_.load(std::memory_order_consume) & ~Flags::dbf_processed;
+  auto flags = flags_.load(std::memory_order_consume) & ~(Flags::dbf_processed | Flags::dbf_moved_handle);
   return create_serialize_tl_object<ton_api::db_block_info>(
       create_tl_block_id(id_), flags, (flags & dbf_inited_prev_left) ? create_tl_block_id(prev_[0]) : nullptr,
       (flags & dbf_inited_prev_right) ? create_tl_block_id(prev_[1]) : nullptr,
       (flags & dbf_inited_next_left) ? create_tl_block_id(next_[0]) : nullptr,
       (flags & dbf_inited_next_right) ? create_tl_block_id(next_[1]) : nullptr, (flags & dbf_inited_lt) ? lt_ : 0,
-      (flags & dbf_inited_ts) ? ts_ : 0, (flags & dbf_inited_state) ? state_ : RootHash::zero());
+      (flags & dbf_inited_ts) ? ts_ : 0, (flags & dbf_inited_state) ? state_ : RootHash::zero(),
+      (flags & dbf_inited_masterchain_ref_block) ? masterchain_ref_seqno_ : 0);
 }
 
 BlockHandleImpl::BlockHandleImpl(td::BufferSlice data) {
   auto obj = fetch_tl_object<ton_api::db_block_info>(std::move(data), true).move_as_ok();
-  flags_ = obj->flags_ & ~Flags::dbf_processed;
+  flags_ = obj->flags_ & ~(Flags::dbf_processed | Flags::dbf_moved_handle);
   id_ = create_block_id(obj->id_);
   prev_[0] = (flags_ & dbf_inited_prev_left) ? create_block_id(obj->prev_left_) : BlockIdExt{};
   prev_[1] = (flags_ & dbf_inited_prev_right) ? create_block_id(obj->prev_right_) : BlockIdExt{};
@@ -52,6 +53,8 @@ BlockHandleImpl::BlockHandleImpl(td::BufferSlice data) {
   lt_ = (flags_ & dbf_inited_lt) ? obj->lt_ : 0;
   ts_ = (flags_ & dbf_inited_ts) ? obj->ts_ : 0;
   state_ = (flags_ & dbf_inited_state) ? obj->state_ : RootHash::zero();
+  masterchain_ref_seqno_ =
+      (flags_ & dbf_inited_masterchain_ref_block) ? static_cast<BlockSeqno>(obj->masterchain_ref_seqno_) : 0;
   get_thread_safe_counter().add(1);
 }
 

validator/block-handle.hpp

@@ -56,11 +56,12 @@ struct BlockHandleImpl : public BlockHandleInterface {
     dbf_inited_state_boc = 0x100000,
     dbf_archived = 0x200000,
     dbf_applied = 0x400000,
-    dbf_moved = 0x1000000,
+    dbf_inited_masterchain_ref_block = 0x800000,
     dbf_deleted = 0x2000000,
     dbf_deleted_boc = 0x4000000,
     dbf_moved_new = 0x8000000,
     dbf_processed = 0x10000000,
+    dbf_moved_handle = 0x20000000,
   };
 
   std::atomic<td::uint64> version_{0};
@@ -72,6 +73,7 @@ struct BlockHandleImpl : public BlockHandleInterface {
   LogicalTime lt_;
   UnixTime ts_;
   RootHash state_;
+  BlockSeqno masterchain_ref_seqno_;
 
   static constexpr td::uint64 lock_const() {
     return static_cast<td::uint64>(1) << 32;
@@ -93,12 +95,12 @@ struct BlockHandleImpl : public BlockHandleInterface {
   bool received() const override {
     return flags_.load(std::memory_order_consume) & Flags::dbf_received;
   }
-  bool moved_to_storage() const override {
-    return flags_.load(std::memory_order_consume) & Flags::dbf_moved;
-  }
   bool moved_to_archive() const override {
     return flags_.load(std::memory_order_consume) & Flags::dbf_moved_new;
   }
+  bool handle_moved_to_archive() const override {
+    return flags_.load(std::memory_order_consume) & Flags::dbf_moved_handle;
+  }
   bool deleted() const override {
     return flags_.load(std::memory_order_consume) & Flags::dbf_deleted;
   }
@@ -199,6 +201,13 @@ struct BlockHandleImpl : public BlockHandleInterface {
   bool is_applied() const override {
     return flags_.load(std::memory_order_consume) & Flags::dbf_applied;
   }
+  bool inited_masterchain_ref_block() const override {
+    return id_.is_masterchain() || (flags_.load(std::memory_order_consume) & Flags::dbf_inited_masterchain_ref_block);
+  }
+  BlockSeqno masterchain_ref_block() const override {
+    CHECK(inited_masterchain_ref_block());
+    return id_.is_masterchain() ? id_.seqno() : masterchain_ref_seqno_;
+  }
   std::vector<BlockIdExt> prev() const override {
     if (is_zero()) {
       return {};
@@ -389,23 +398,17 @@ struct BlockHandleImpl : public BlockHandleInterface {
     flags_ |= Flags::dbf_received;
     unlock();
   }
-  void set_moved_to_storage() override {
-    if (flags_.load(std::memory_order_consume) & Flags::dbf_moved) {
-      return;
-    }
-    lock();
-    flags_ |= Flags::dbf_moved;
-    unlock();
-  }
   void set_moved_to_archive() override {
     if (flags_.load(std::memory_order_consume) & Flags::dbf_moved_new) {
       return;
     }
     lock();
     flags_ |= Flags::dbf_moved_new;
-    flags_ &= ~Flags::dbf_moved;
     unlock();
   }
+  void set_handle_moved_to_archive() override {
+    flags_ |= Flags::dbf_moved_handle;
+  }
   void set_deleted() override {
     if (flags_.load(std::memory_order_consume) & Flags::dbf_deleted) {
       return;
@@ -485,6 +488,14 @@ struct BlockHandleImpl : public BlockHandleInterface {
       unlock();
     }
   }
+  void set_masterchain_ref_block(BlockSeqno seqno) override {
+    if (!inited_masterchain_ref_block()) {
+      lock();
+      masterchain_ref_seqno_ = seqno;
+      flags_ |= Flags::dbf_inited_masterchain_ref_block;
+      unlock();
+    }
+  }
 
   void unsafe_clear_applied() override {
     if (is_applied()) {

validator/db/archive-manager.cpp

@@ -0,0 +1,986 @@
+#include "archive-manager.hpp"
+#include "td/actor/MultiPromise.h"
+#include "td/utils/overloaded.h"
+#include "files-async.hpp"
+#include "td/db/RocksDb.h"
+#include "common/delay.h"
+
+namespace ton {
+
+namespace validator {
+
+std::string PackageId::path() const {
+  if (temp) {
+    return "files/packages/";
+  } else if (key) {
+    char s[24];
+    sprintf(s, "key%03d", id / 1000000);
+    return PSTRING() << "archive/packages/" << s << "/";
+  } else {
+    char s[20];
+    sprintf(s, "arch%04d", id / 100000);
+    return PSTRING() << "archive/packages/" << s << "/";
+  }
+}
+
+std::string PackageId::name() const {
+  if (temp) {
+    return PSTRING() << "temp.archive." << id;
+  } else if (key) {
+    char s[20];
+    sprintf(s, "%06d", id);
+    return PSTRING() << "key.archive." << s;
+  } else {
+    char s[10];
+    sprintf(s, "%05d", id);
+    return PSTRING() << "archive." << s;
+  }
+}
+
+ArchiveManager::ArchiveManager(td::actor::ActorId<RootDb> root, std::string db_root) : db_root_(db_root) {
+}
+
+void ArchiveManager::add_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
+  if (handle->handle_moved_to_archive()) {
+    update_handle(std::move(handle), std::move(promise));
+    return;
+  }
+  auto p = get_package_id_force(handle->masterchain_ref_block(), handle->id().shard_full(), handle->id().seqno(),
+                                handle->unix_time(), handle->logical_time(),
+                                handle->inited_is_key_block() && handle->is_key_block());
+  auto f = get_file_desc(handle->id().shard_full(), p, handle->id().seqno(), handle->unix_time(),
+                         handle->logical_time(), true);
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_handle, std::move(handle), std::move(promise));
+}
+
+void ArchiveManager::update_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
+  FileDescription *f;
+  if (handle->handle_moved_to_archive()) {
+    CHECK(handle->inited_unix_time());
+    f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), handle->id().seqno(),
+                      handle->unix_time(), handle->logical_time(), true);
+  } else {
+    f = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true);
+  }
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::update_handle, std::move(handle), std::move(promise));
+}
+
+void ArchiveManager::add_file(BlockHandle handle, FileReference ref_id, td::BufferSlice data,
+                              td::Promise<td::Unit> promise) {
+  bool copy_to_key = false;
+  if (handle->inited_is_key_block() && handle->is_key_block() && handle->inited_unix_time() &&
+      handle->inited_logical_time() && handle->inited_masterchain_ref_block()) {
+    copy_to_key = (ref_id.ref().get_offset() == ref_id.ref().offset<fileref::Proof>()) ||
+                  (ref_id.ref().get_offset() == ref_id.ref().offset<fileref::ProofLink>());
+  }
+  if (!handle->handle_moved_to_archive()) {
+    td::MultiPromise mp;
+    auto ig = mp.init_guard();
+    ig.add_promise(std::move(promise));
+    auto f1 = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true);
+    td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), data.clone(),
+                            ig.get_promise());
+    if (copy_to_key) {
+      auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()),
+                              handle->id().seqno(), handle->unix_time(), handle->logical_time(), true);
+      td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, ref_id, std::move(data), ig.get_promise());
+    }
+    return;
+  }
+
+  CHECK(handle->inited_is_key_block());
+
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise(std::move(promise));
+  auto f1 = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()),
+                          handle->id().seqno(), handle->unix_time(), handle->logical_time(), true);
+  td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, ref_id, data.clone(), ig.get_promise());
+  if (copy_to_key) {
+    auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()),
+                            handle->id().seqno(), handle->unix_time(), handle->logical_time(), true);
+    td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, ref_id, std::move(data), ig.get_promise());
+  }
+}
+
+void ArchiveManager::add_key_block_proof(UnixTime ts, BlockSeqno seqno, LogicalTime lt, FileReference ref_id,
+                                         td::BufferSlice data, td::Promise<td::Unit> promise) {
+  auto f = get_file_desc(ShardIdFull{masterchainId}, get_key_package_id(seqno), seqno, ts, lt, true);
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), std::move(data),
+                          std::move(promise));
+}
+
+void ArchiveManager::add_temp_file_short(FileReference ref_id, td::BufferSlice data, td::Promise<td::Unit> promise) {
+  auto f = get_file_desc(ref_id.shard(), get_temp_package_id(), 0, 0, 0, true);
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), std::move(data),
+                          std::move(promise));
+}
+
+void ArchiveManager::get_handle(BlockIdExt block_id, td::Promise<BlockHandle> promise) {
+  auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), false);
+  if (f) {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, idx = get_max_temp_file_desc_idx(),
+                                         promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+      if (R.is_ok()) {
+        promise.set_value(R.move_as_ok());
+      } else {
+        td::actor::send_closure(SelfId, &ArchiveManager::get_handle_cont, block_id, idx, std::move(promise));
+      }
+    });
+    td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_handle, block_id, std::move(P));
+  } else {
+    get_handle_cont(block_id, get_max_temp_file_desc_idx(), std::move(promise));
+  }
+}
+
+void ArchiveManager::get_handle_cont(BlockIdExt block_id, PackageId idx, td::Promise<BlockHandle> promise) {
+  if (idx.is_empty()) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "block handle not in db"));
+    return;
+  }
+  auto f = get_temp_file_desc_by_idx(idx);
+  if (!f) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "block handle not in db"));
+    return;
+  }
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, idx = get_prev_temp_file_desc_idx(idx),
+                                       promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+    if (R.is_ok()) {
+      td::actor::send_closure(SelfId, &ArchiveManager::get_handle_finish, R.move_as_ok(), std::move(promise));
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveManager::get_handle_cont, block_id, idx, std::move(promise));
+    }
+  });
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_handle, block_id, std::move(P));
+}
+
+void ArchiveManager::get_handle_finish(BlockHandle handle, td::Promise<BlockHandle> promise) {
+  auto f = get_file_desc_by_seqno(handle->id().shard_full(), handle->id().seqno(), false);
+  if (!f) {
+    promise.set_value(std::move(handle));
+    return;
+  }
+  auto P = td::PromiseCreator::lambda([handle, promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+    if (R.is_ok()) {
+      promise.set_value(R.move_as_ok());
+    } else {
+      promise.set_value(std::move(handle));
+    }
+  });
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_handle, handle->id(), std::move(P));
+}
+
+void ArchiveManager::get_file_short(FileReference ref_id, td::Promise<td::BufferSlice> promise) {
+  bool search_in_key = false;
+  BlockIdExt block_id;
+  ref_id.ref().visit(td::overloaded(
+      [&](const fileref::Proof &p) {
+        search_in_key = p.block_id.is_masterchain();
+        block_id = p.block_id;
+      },
+      [&](const fileref::ProofLink &p) {
+        search_in_key = p.block_id.is_masterchain();
+        block_id = p.block_id;
+      },
+      [&](const auto &p) {}));
+  if (search_in_key) {
+    auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), true);
+    if (f) {
+      auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id,
+                                           promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+        if (R.is_ok()) {
+          promise.set_value(R.move_as_ok());
+        } else {
+          td::actor::send_closure(SelfId, &ArchiveManager::get_temp_file_short, std::move(ref_id), std::move(promise));
+        }
+      });
+      td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, ref_id, std::move(P));
+      return;
+    }
+  }
+  get_temp_file_short(std::move(ref_id), std::move(promise));
+}
+
+void ArchiveManager::get_key_block_proof(FileReference ref_id, td::Promise<td::BufferSlice> promise) {
+  bool search_in_key = false;
+  BlockIdExt block_id;
+  ref_id.ref().visit(td::overloaded(
+      [&](const fileref::Proof &p) {
+        search_in_key = p.block_id.is_masterchain();
+        block_id = p.block_id;
+      },
+      [&](const fileref::ProofLink &p) {
+        search_in_key = p.block_id.is_masterchain();
+        block_id = p.block_id;
+      },
+      [&](const auto &p) {}));
+  if (search_in_key) {
+    auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), true);
+    if (f) {
+      td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, ref_id, std::move(promise));
+    } else {
+      promise.set_error(td::Status::Error(ErrorCode::notready, "key proof not in db"));
+    }
+  } else {
+    promise.set_error(
+        td::Status::Error(ErrorCode::protoviolation, "only proof/prooflink supported in get_key_block_proof"));
+  }
+}
+
+void ArchiveManager::get_temp_file_short(FileReference ref_id, td::Promise<td::BufferSlice> promise) {
+  get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise));
+}
+
+void ArchiveManager::get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise<td::BufferSlice> promise) {
+  auto f = get_temp_file_desc_by_idx(idx);
+  if (!f) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "file not in db"));
+    return;
+  }
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id, idx = get_prev_temp_file_desc_idx(idx),
+                                       promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_ok()) {
+      promise.set_value(R.move_as_ok());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveManager::get_file_short_cont, std::move(ref_id), idx, std::move(promise));
+    }
+  });
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, std::move(ref_id), std::move(P));
+}
+
+void ArchiveManager::get_file(BlockHandle handle, FileReference ref_id, td::Promise<td::BufferSlice> promise) {
+  if (handle->moved_to_archive()) {
+    auto f = get_file_desc(handle->id().shard_full(), PackageId{handle->masterchain_ref_block(), false, false}, 0, 0, 0,
+                           false);
+    if (f) {
+      td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, std::move(ref_id), std::move(promise));
+      return;
+    }
+  }
+  get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise));
+}
+
+void ArchiveManager::written_perm_state(FileReferenceShort id) {
+  perm_states_.emplace(id.hash(), id);
+}
+
+void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise<td::Unit> promise) {
+  auto id = FileReference{fileref::ZeroState{block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) != perm_states_.end()) {
+    promise.set_value(td::Unit());
+    return;
+  }
+
+  auto path = db_root_ + "/archive/states/" + id.filename_short();
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), id = id.shortref(), promise = std::move(promise)](td::Result<std::string> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id);
+          promise.set_value(td::Unit());
+        }
+      });
+  td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/", path, std::move(data), std::move(P))
+      .release();
+}
+
+void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data,
+                                          td::Promise<td::Unit> promise) {
+  auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) != perm_states_.end()) {
+    promise.set_value(td::Unit());
+    return;
+  }
+
+  auto path = db_root_ + "/archive/states/" + id.filename_short();
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), id = id.shortref(), promise = std::move(promise)](td::Result<std::string> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id);
+          promise.set_value(td::Unit());
+        }
+      });
+  td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/", path, std::move(data), std::move(P))
+      .release();
+}
+
+void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {
+  auto id = FileReference{fileref::ZeroState{block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) == perm_states_.end()) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "zerostate not in db"));
+    return;
+  }
+
+  auto path = db_root_ + "/archive/states/" + id.filename_short();
+  td::actor::create_actor<db::ReadFile>("readfile", path, 0, -1, 0, std::move(promise)).release();
+}
+
+void ArchiveManager::check_zero_state(BlockIdExt block_id, td::Promise<bool> promise) {
+  auto id = FileReference{fileref::ZeroState{block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) == perm_states_.end()) {
+    promise.set_result(false);
+    return;
+  }
+  promise.set_result(true);
+}
+
+void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                          td::Promise<td::BufferSlice> promise) {
+  auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) == perm_states_.end()) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db"));
+    return;
+  }
+
+  auto path = db_root_ + "/archive/states/" + id.filename_short();
+  td::actor::create_actor<db::ReadFile>("readfile", path, 0, -1, 0, std::move(promise)).release();
+}
+
+void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,
+                                                td::int64 max_size, td::Promise<td::BufferSlice> promise) {
+  auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) == perm_states_.end()) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db"));
+    return;
+  }
+
+  auto path = db_root_ + "/archive/states/" + id.filename_short();
+  td::actor::create_actor<db::ReadFile>("readfile", path, offset, max_size, 0, std::move(promise)).release();
+}
+
+void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                            td::Promise<bool> promise) {
+  auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) == perm_states_.end()) {
+    promise.set_result(false);
+    return;
+  }
+  promise.set_result(true);
+}
+
+void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts,
+                                            td::Promise<BlockHandle> promise) {
+  auto f = get_file_desc_by_unix_time(account_id, ts, false);
+  if (f) {
+    td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_unix_time, account_id, ts,
+                            std::move(promise));
+  } else {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "ts not in db"));
+  }
+}
+
+void ArchiveManager::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockHandle> promise) {
+  auto f = get_file_desc_by_lt(account_id, lt, false);
+  if (f) {
+    td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(promise));
+  } else {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "lt not in db"));
+  }
+}
+
+void ArchiveManager::get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno,
+                                        td::Promise<BlockHandle> promise) {
+  auto f = get_file_desc_by_seqno(account_id, seqno, false);
+  if (f) {
+    td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_seqno, account_id, seqno,
+                            std::move(promise));
+  } else {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "seqno not in db"));
+  }
+}
+
+void ArchiveManager::delete_package(PackageId id) {
+  auto key = create_serialize_tl_object<ton_api::db_files_package_key>(id.id, id.key, id.temp);
+
+  std::string value;
+  auto v = index_->get(key.as_slice(), value);
+  v.ensure();
+  CHECK(v.move_as_ok() == td::KeyValue::GetStatus::Ok);
+
+  auto R = fetch_tl_object<ton_api::db_files_package_value>(value, true);
+  R.ensure();
+  auto x = R.move_as_ok();
+
+  if (x->deleted_) {
+    return;
+  }
+
+  auto &m = get_file_map(id);
+  auto it = m.find(id);
+  if (it == m.end() || it->second.deleted) {
+    return;
+  }
+
+  it->second.deleted = true;
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id](td::Result<td::Unit> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveManager::deleted_package, id);
+  });
+  td::actor::send_closure(it->second.file_actor_id(), &ArchiveSlice::destroy, std::move(P));
+}
+
+void ArchiveManager::deleted_package(PackageId id) {
+  auto key = create_serialize_tl_object<ton_api::db_files_package_key>(id.id, id.key, id.temp);
+
+  std::string value;
+  auto v = index_->get(key.as_slice(), value);
+  v.ensure();
+  CHECK(v.move_as_ok() == td::KeyValue::GetStatus::Ok);
+
+  auto R = fetch_tl_object<ton_api::db_files_package_value>(value, true);
+  R.ensure();
+  auto x = R.move_as_ok();
+
+  if (x->deleted_) {
+    return;
+  }
+  x->deleted_ = true;
+  index_->begin_transaction().ensure();
+  index_->set(key, serialize_tl_object(x, true)).ensure();
+  index_->commit_transaction().ensure();
+
+  auto &m = get_file_map(id);
+  auto it = m.find(id);
+  CHECK(it != m.end());
+  CHECK(it->second.deleted);
+  it->second.clear_actor_id();
+}
+
+void ArchiveManager::load_package(PackageId id) {
+  auto key = create_serialize_tl_object<ton_api::db_files_package_key>(id.id, id.key, id.temp);
+
+  std::string value;
+  auto v = index_->get(key.as_slice(), value);
+  v.ensure();
+  CHECK(v.move_as_ok() == td::KeyValue::GetStatus::Ok);
+
+  auto R = fetch_tl_object<ton_api::db_files_package_value>(value, true);
+  R.ensure();
+  auto x = R.move_as_ok();
+
+  if (x->deleted_) {
+    return;
+  }
+
+  FileDescription desc{id, false};
+  if (!id.temp) {
+    for (auto &e : x->firstblocks_) {
+      desc.first_blocks[ShardIdFull{e->workchain_, static_cast<ShardId>(e->shard_)}] = FileDescription::Desc{
+          static_cast<BlockSeqno>(e->seqno_), static_cast<UnixTime>(e->unixtime_), static_cast<LogicalTime>(e->lt_)};
+    }
+  }
+
+  std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
+  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.key, id.temp, prefix);
+
+  get_file_map(id).emplace(id, std::move(desc));
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno,
+                                                               UnixTime ts, LogicalTime lt, bool force) {
+  auto &f = get_file_map(id);
+  auto it = f.find(id);
+  if (it != f.end()) {
+    if (it->second.deleted) {
+      CHECK(!force);
+      return nullptr;
+    }
+    if (force && !id.temp) {
+      update_desc(it->second, shard, seqno, ts, lt);
+    }
+    return &it->second;
+  }
+  if (!force) {
+    return nullptr;
+  }
+
+  return add_file_desc(shard, id, seqno, ts, lt);
+}
+
+ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno,
+                                                               UnixTime ts, LogicalTime lt) {
+  auto &f = get_file_map(id);
+  CHECK(f.count(id) == 0);
+
+  FileDescription desc{id, false};
+  td::mkdir(db_root_ + id.path()).ensure();
+  std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
+  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.key, id.temp, prefix);
+  if (!id.temp) {
+    update_desc(desc, shard, seqno, ts, lt);
+  }
+
+  std::vector<tl_object_ptr<ton_api::db_files_package_firstBlock>> vec;
+  for (auto &e : desc.first_blocks) {
+    vec.push_back(create_tl_object<ton_api::db_files_package_firstBlock>(e.first.workchain, e.first.shard,
+                                                                         e.second.seqno, e.second.ts, e.second.lt));
+  }
+
+  index_->begin_transaction().ensure();
+  // add package info to list of packages
+  {
+    std::vector<td::int32> t;
+    std::vector<td::int32> tk;
+    std::vector<td::int32> tt;
+    for (auto &e : files_) {
+      t.push_back(e.first.id);
+    }
+    for (auto &e : key_files_) {
+      tk.push_back(e.first.id);
+    }
+    for (auto &e : temp_files_) {
+      tt.push_back(e.first.id);
+    }
+    (id.temp ? tt : (id.key ? tk : t)).push_back(id.id);
+    index_
+        ->set(create_serialize_tl_object<ton_api::db_files_index_key>().as_slice(),
+              create_serialize_tl_object<ton_api::db_files_index_value>(std::move(t), std::move(tk), std::move(tt))
+                  .as_slice())
+        .ensure();
+  }
+  // add package info key
+  {
+    index_
+        ->set(create_serialize_tl_object<ton_api::db_files_package_key>(id.id, id.key, id.temp).as_slice(),
+              create_serialize_tl_object<ton_api::db_files_package_value>(id.id, id.key, id.temp, std::move(vec), false)
+                  .as_slice())
+        .ensure();
+  }
+  index_->commit_transaction().ensure();
+
+  return &f.emplace(id, std::move(desc)).first->second;
+}
+
+void ArchiveManager::update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts,
+                                 LogicalTime lt) {
+  auto it = desc.first_blocks.find(shard);
+  if (it != desc.first_blocks.end() && it->second.seqno <= seqno) {
+    return;
+  }
+  desc.first_blocks[shard] = FileDescription::Desc{seqno, ts, lt};
+  std::vector<tl_object_ptr<ton_api::db_files_package_firstBlock>> vec;
+  for (auto &e : desc.first_blocks) {
+    vec.push_back(create_tl_object<ton_api::db_files_package_firstBlock>(e.first.workchain, e.first.shard,
+                                                                         e.second.seqno, e.second.ts, e.second.lt));
+  }
+  index_->begin_transaction().ensure();
+  index_
+      ->set(create_serialize_tl_object<ton_api::db_files_package_key>(desc.id.id, desc.id.key, desc.id.temp).as_slice(),
+            create_serialize_tl_object<ton_api::db_files_package_value>(desc.id.id, desc.id.key, desc.id.temp,
+                                                                        std::move(vec), false)
+                .as_slice())
+      .ensure();
+  index_->commit_transaction().ensure();
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno,
+                                                                        bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    auto i = it->second.first_blocks.find(shard);
+    if (i != it->second.first_blocks.end() && i->second.seqno <= seqno) {
+      return &it->second;
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts,
+                                                                            bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    auto i = it->second.first_blocks.find(shard);
+    if (i != it->second.first_blocks.end() && i->second.ts <= ts) {
+      return &it->second;
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt,
+                                                                     bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    auto i = it->second.first_blocks.find(shard);
+    if (i != it->second.first_blocks.end() && i->second.lt <= lt) {
+      return &it->second;
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno,
+                                                                        bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  if (account.is_masterchain()) {
+    return get_file_desc_by_seqno(ShardIdFull{masterchainId}, seqno, key_block);
+  }
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    bool found = false;
+    for (int i = 0; i < 60; i++) {
+      auto shard = shard_prefix(account, i);
+      auto it2 = it->second.first_blocks.find(shard);
+      if (it2 != it->second.first_blocks.end()) {
+        if (it2->second.seqno <= seqno) {
+          return &it->second;
+        }
+        found = true;
+      } else if (found) {
+        break;
+      }
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(AccountIdPrefixFull account, UnixTime ts,
+                                                                            bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  if (account.is_masterchain()) {
+    return get_file_desc_by_unix_time(ShardIdFull{masterchainId}, ts, key_block);
+  }
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    bool found = false;
+    for (int i = 0; i < 60; i++) {
+      auto shard = shard_prefix(account, i);
+      auto it2 = it->second.first_blocks.find(shard);
+      if (it2 != it->second.first_blocks.end()) {
+        if (it2->second.ts <= ts) {
+          return &it->second;
+        }
+        found = true;
+      } else if (found) {
+        break;
+      }
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(AccountIdPrefixFull account, LogicalTime lt,
+                                                                     bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  if (account.is_masterchain()) {
+    return get_file_desc_by_lt(ShardIdFull{masterchainId}, lt, key_block);
+  }
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    bool found = false;
+    for (int i = 0; i < 60; i++) {
+      auto shard = shard_prefix(account, i);
+      auto it2 = it->second.first_blocks.find(shard);
+      if (it2 != it->second.first_blocks.end()) {
+        if (it2->second.lt <= lt) {
+          return &it->second;
+        }
+        found = true;
+      } else if (found) {
+        break;
+      }
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_temp_file_desc_by_idx(PackageId idx) {
+  auto it = temp_files_.find(idx);
+  if (it != temp_files_.end()) {
+    if (it->second.deleted) {
+      return nullptr;
+    } else {
+      return &it->second;
+    }
+  } else {
+    return nullptr;
+  }
+}
+
+PackageId ArchiveManager::get_max_temp_file_desc_idx() {
+  if (temp_files_.size() == 0) {
+    return PackageId::empty(false, true);
+  } else {
+    return temp_files_.rbegin()->first;
+  }
+}
+
+PackageId ArchiveManager::get_prev_temp_file_desc_idx(PackageId idx) {
+  auto it = temp_files_.lower_bound(idx);
+  if (it == temp_files_.end()) {
+    return PackageId::empty(false, true);
+  }
+  if (it == temp_files_.begin()) {
+    return PackageId::empty(false, true);
+  }
+  it--;
+  return it->first;
+}
+
+void ArchiveManager::start_up() {
+  td::mkdir(db_root_).ensure();
+  td::mkdir(db_root_ + "/archive/").ensure();
+  td::mkdir(db_root_ + "/archive/tmp/").ensure();
+  td::mkdir(db_root_ + "/archive/packages/").ensure();
+  td::mkdir(db_root_ + "/archive/states/").ensure();
+  td::mkdir(db_root_ + "/files/").ensure();
+  td::mkdir(db_root_ + "/files/packages/").ensure();
+  index_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_root_ + "/files/globalindex").move_as_ok());
+  std::string value;
+  auto v = index_->get(create_serialize_tl_object<ton_api::db_files_index_key>().as_slice(), value);
+  v.ensure();
+  if (v.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+    auto R = fetch_tl_object<ton_api::db_files_index_value>(value, true);
+    R.ensure();
+    auto x = R.move_as_ok();
+
+    for (auto &d : x->packages_) {
+      load_package(PackageId{static_cast<td::uint32>(d), false, false});
+    }
+    for (auto &d : x->key_packages_) {
+      load_package(PackageId{static_cast<td::uint32>(d), true, false});
+    }
+    for (auto &d : x->temp_packages_) {
+      load_package(PackageId{static_cast<td::uint32>(d), false, true});
+    }
+  }
+
+  td::WalkPath::run(db_root_ + "/archive/states/", [&](td::CSlice fname, td::WalkPath::Type t) -> void {
+    if (t == td::WalkPath::Type::NotDir) {
+      LOG(ERROR) << "checking file " << fname;
+      auto pos = fname.rfind('/');
+      if (pos != td::Slice::npos) {
+        fname.remove_prefix(pos + 1);
+      }
+      auto R = FileReferenceShort::create(fname.str());
+      if (R.is_error()) {
+        auto R2 = FileReference::create(fname.str());
+        if (R2.is_error()) {
+          LOG(ERROR) << "deleting bad state file '" << fname << "': " << R.move_as_error() << R2.move_as_error();
+          td::unlink(db_root_ + "/archive/states/" + fname.str()).ignore();
+          return;
+        }
+        auto d = R2.move_as_ok();
+        auto newfname = d.filename_short();
+        td::rename(db_root_ + "/archive/states/" + fname.str(), db_root_ + "/archive/states/" + newfname).ensure();
+        R = FileReferenceShort::create(newfname);
+        R.ensure();
+      }
+      auto f = R.move_as_ok();
+      auto hash = f.hash();
+      perm_states_[hash] = std::move(f);
+    }
+  }).ensure();
+
+  persistent_state_gc(FileHash::zero());
+}
+
+void ArchiveManager::run_gc(UnixTime ts) {
+  auto p = get_temp_package_id_by_unixtime(ts);
+  std::vector<PackageId> vec;
+  for (auto &x : temp_files_) {
+    if (x.first < p) {
+      vec.push_back(x.first);
+    } else {
+      break;
+    }
+  }
+  if (vec.size() <= 1) {
+    return;
+  }
+  vec.resize(vec.size() - 1, PackageId::empty(false, true));
+
+  for (auto &x : vec) {
+    delete_package(x);
+  }
+}
+
+void ArchiveManager::persistent_state_gc(FileHash last) {
+  if (perm_states_.size() == 0) {
+    delay_action(
+        [hash = FileHash::zero(), SelfId = actor_id(this)]() {
+          td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash);
+        },
+        td::Timestamp::in(1.0));
+    return;
+  }
+  auto it = perm_states_.lower_bound(last);
+  if (it != perm_states_.end() && it->first == last) {
+    it++;
+  }
+  if (it == perm_states_.end()) {
+    it = perm_states_.begin();
+  }
+
+  auto &F = it->second;
+  auto hash = F.hash();
+
+  int res = 0;
+  BlockSeqno seqno = 0;
+  F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &x) { res = 1; },
+                               [&](const fileref::PersistentStateShort &x) {
+                                 res = 0;
+                                 seqno = x.masterchain_seqno;
+                               },
+                               [&](const auto &obj) { res = -1; }));
+
+  if (res == -1) {
+    td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore();
+    perm_states_.erase(it);
+  }
+  if (res != 0) {
+    delay_action([hash, SelfId = actor_id(
+                            this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); },
+                 td::Timestamp::in(1.0));
+    return;
+  }
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), hash](td::Result<BlockHandle> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, nullptr, hash);
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, R.move_as_ok(), hash);
+    }
+  });
+
+  get_block_by_seqno(AccountIdPrefixFull{masterchainId, 0}, seqno, std::move(P));
+}
+
+void ArchiveManager::got_gc_masterchain_handle(BlockHandle handle, FileHash hash) {
+  bool to_del = false;
+  if (!handle || !handle->inited_unix_time() || !handle->unix_time()) {
+    to_del = true;
+  } else {
+    auto ttl = ValidatorManager::persistent_state_ttl(handle->unix_time());
+    to_del = ttl < td::Clocks::system();
+  }
+  auto it = perm_states_.find(hash);
+  CHECK(it != perm_states_.end());
+  auto &F = it->second;
+  if (to_del) {
+    td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore();
+    perm_states_.erase(it);
+  }
+  delay_action([hash, SelfId = actor_id(
+                          this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); },
+               td::Timestamp::in(1.0));
+}
+
+PackageId ArchiveManager::get_temp_package_id() const {
+  return get_temp_package_id_by_unixtime(static_cast<UnixTime>(td::Clocks::system()));
+}
+
+PackageId ArchiveManager::get_temp_package_id_by_unixtime(UnixTime ts) const {
+  return PackageId{ts - (ts % 3600), false, true};
+}
+
+PackageId ArchiveManager::get_key_package_id(BlockSeqno seqno) const {
+  return PackageId{seqno - seqno % 200000, true, false};
+}
+
+PackageId ArchiveManager::get_package_id(BlockSeqno seqno) const {
+  auto it = files_.upper_bound(PackageId{seqno, false, false});
+  CHECK(it != files_.begin());
+  it--;
+  return it->first;
+}
+
+PackageId ArchiveManager::get_package_id_force(BlockSeqno masterchain_seqno, ShardIdFull shard, BlockSeqno seqno,
+                                               UnixTime ts, LogicalTime lt, bool is_key) {
+  PackageId p = PackageId::empty(false, false);
+  if (!is_key) {
+    auto it = files_.upper_bound(PackageId{masterchain_seqno, false, false});
+    p = PackageId{masterchain_seqno - (masterchain_seqno % 20000), false, false};
+    if (it != files_.begin()) {
+      it--;
+      if (p < it->first) {
+        p = it->first;
+      }
+    }
+  } else {
+    p = PackageId{masterchain_seqno, false, false};
+  }
+  auto it = files_.find(p);
+  if (it != files_.end()) {
+    return it->first;
+  }
+  add_file_desc(shard, p, seqno, ts, lt);
+  it = files_.find(p);
+  CHECK(it != files_.end());
+  return it->first;
+}
+
+void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) {
+  auto F = get_file_desc_by_seqno(ShardIdFull{masterchainId}, masterchain_seqno, false);
+  if (!F) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found"));
+    return;
+  }
+
+  promise.set_result(F->id.id);
+}
+
+void ArchiveManager::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                                       td::Promise<td::BufferSlice> promise) {
+  if (archive_id != static_cast<td::uint32>(archive_id)) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found"));
+    return;
+  }
+  auto F = get_file_desc(ShardIdFull{masterchainId}, PackageId{static_cast<BlockSeqno>(archive_id), false, false}, 0, 0,
+                         0, false);
+  if (!F) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found"));
+    return;
+  }
+
+  td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_slice, offset, limit, std::move(promise));
+}
+
+void ArchiveManager::commit_transaction() {
+  if (!async_mode_ || huge_transaction_size_++ >= 100) {
+    index_->commit_transaction().ensure();
+    if (async_mode_) {
+      huge_transaction_size_ = 0;
+      huge_transaction_started_ = false;
+    }
+  }
+}
+
+void ArchiveManager::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
+  async_mode_ = mode;
+  if (!async_mode_ && huge_transaction_started_) {
+    index_->commit_transaction().ensure();
+    huge_transaction_size_ = 0;
+    huge_transaction_started_ = false;
+  }
+
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise(std::move(promise));
+
+  for (auto &x : key_files_) {
+    if (!x.second.deleted) {
+      td::actor::send_closure(x.second.file_actor_id(), &ArchiveSlice::set_async_mode, mode, ig.get_promise());
+    }
+  }
+  for (auto &x : temp_files_) {
+    if (!x.second.deleted) {
+      td::actor::send_closure(x.second.file_actor_id(), &ArchiveSlice::set_async_mode, mode, ig.get_promise());
+    }
+  }
+  for (auto &x : files_) {
+    if (!x.second.deleted) {
+      td::actor::send_closure(x.second.file_actor_id(), &ArchiveSlice::set_async_mode, mode, ig.get_promise());
+    }
+  }
+}
+}  // namespace validator
+
+}  // namespace ton

validator/db/archive-manager.hpp

@@ -0,0 +1,155 @@
+#pragma once
+
+#include "archive-slice.hpp"
+
+namespace ton {
+
+namespace validator {
+
+struct PackageId {
+  td::uint32 id;
+  bool key;
+  bool temp;
+
+  explicit PackageId(td::uint32 id, bool key, bool temp) : id(id), key(key), temp(temp) {
+  }
+
+  bool operator<(const PackageId &with) const {
+    return id < with.id;
+  }
+  bool operator==(const PackageId &with) const {
+    return id == with.id;
+  }
+
+  std::string path() const;
+  std::string name() const;
+
+  bool is_empty() {
+    return id == std::numeric_limits<td::uint32>::max();
+  }
+  static PackageId empty(bool key, bool temp) {
+    return PackageId(std::numeric_limits<td::uint32>::max(), key, temp);
+  }
+};
+
+class RootDb;
+
+class ArchiveManager : public td::actor::Actor {
+ public:
+  ArchiveManager(td::actor::ActorId<RootDb> root, std::string db_root);
+
+  void add_handle(BlockHandle handle, td::Promise<td::Unit> promise);
+  void update_handle(BlockHandle handle, td::Promise<td::Unit> promise);
+  void add_file(BlockHandle handle, FileReference ref_id, td::BufferSlice data, td::Promise<td::Unit> promise);
+  void add_key_block_proof(BlockSeqno seqno, UnixTime ts, LogicalTime lt, FileReference ref_id, td::BufferSlice data,
+                           td::Promise<td::Unit> promise);
+  void add_temp_file_short(FileReference ref_id, td::BufferSlice data, td::Promise<td::Unit> promise);
+  void get_handle(BlockIdExt block_id, td::Promise<BlockHandle> promise);
+  void get_key_block_proof(FileReference ref_id, td::Promise<td::BufferSlice> promise);
+  void get_temp_file_short(FileReference ref_id, td::Promise<td::BufferSlice> promise);
+  void get_file_short(FileReference ref_id, td::Promise<td::BufferSlice> promise);
+  void get_file(BlockHandle handle, FileReference ref_id, td::Promise<td::BufferSlice> promise);
+
+  void add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise<td::Unit> promise);
+  void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data,
+                            td::Promise<td::Unit> promise);
+  void get_zero_state(BlockIdExt block_id, td::Promise<td::BufferSlice> promise);
+  void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::BufferSlice> promise);
+  void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,
+                                  td::int64 max_size, td::Promise<td::BufferSlice> promise);
+  void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<bool> promise);
+  void check_zero_state(BlockIdExt block_id, td::Promise<bool> promise);
+
+  void run_gc(UnixTime ts);
+
+  /* from LTDB */
+  void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise<BlockHandle> promise);
+  void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockHandle> promise);
+  void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise<BlockHandle> promise);
+
+  void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise);
+  void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                         td::Promise<td::BufferSlice> promise);
+
+  void start_up() override;
+
+  void begin_transaction();
+  void commit_transaction();
+  void set_async_mode(bool mode, td::Promise<td::Unit> promise);
+
+ private:
+  struct FileDescription {
+    struct Desc {
+      BlockSeqno seqno;
+      UnixTime ts;
+      LogicalTime lt;
+    };
+    FileDescription(PackageId id, bool deleted) : id(id), deleted(deleted) {
+    }
+    auto file_actor_id() const {
+      return file.get();
+    }
+    void clear_actor_id() {
+      file.reset();
+    }
+    PackageId id;
+    bool deleted;
+
+    std::map<ShardIdFull, Desc> first_blocks;
+    td::actor::ActorOwn<ArchiveSlice> file;
+  };
+
+  std::map<PackageId, FileDescription> files_;
+  std::map<PackageId, FileDescription> key_files_;
+  std::map<PackageId, FileDescription> temp_files_;
+  bool async_mode_ = false;
+  bool huge_transaction_started_ = false;
+  td::uint32 huge_transaction_size_ = 0;
+
+  auto &get_file_map(const PackageId &p) {
+    return p.key ? key_files_ : p.temp ? temp_files_ : files_;
+  }
+
+  std::map<FileHash, FileReferenceShort> perm_states_;
+
+  void load_package(PackageId seqno);
+  void delete_package(PackageId seqno);
+  void deleted_package(PackageId seqno);
+  void get_handle_cont(BlockIdExt block_id, PackageId id, td::Promise<BlockHandle> promise);
+  void get_handle_finish(BlockHandle handle, td::Promise<BlockHandle> promise);
+  void get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise<td::BufferSlice> promise);
+
+  FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt,
+                                 bool force);
+  FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt);
+  void update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, LogicalTime lt);
+  FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, bool key_block);
+  FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, bool key_block);
+  FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, bool key_block);
+  FileDescription *get_file_desc_by_seqno(AccountIdPrefixFull shard, BlockSeqno seqno, bool key_block);
+  FileDescription *get_file_desc_by_lt(AccountIdPrefixFull shard, LogicalTime lt, bool key_block);
+  FileDescription *get_file_desc_by_unix_time(AccountIdPrefixFull shard, UnixTime ts, bool key_block);
+  FileDescription *get_temp_file_desc_by_idx(PackageId idx);
+  PackageId get_max_temp_file_desc_idx();
+  PackageId get_prev_temp_file_desc_idx(PackageId id);
+
+  void written_perm_state(FileReferenceShort id);
+
+  void persistent_state_gc(FileHash last);
+  void got_gc_masterchain_handle(BlockHandle handle, FileHash hash);
+
+  std::string db_root_;
+
+  std::shared_ptr<td::KeyValue> index_;
+
+  PackageId get_package_id(BlockSeqno seqno) const;
+  PackageId get_package_id_force(BlockSeqno masterchain_seqno, ShardIdFull shard, BlockSeqno seqno, UnixTime ts,
+                                 LogicalTime lt, bool is_key);
+  PackageId get_temp_package_id() const;
+  PackageId get_key_package_id(BlockSeqno seqno) const;
+  PackageId get_temp_package_id_by_unixtime(UnixTime ts) const;
+};
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/archive-mover.cpp

@@ -0,0 +1,475 @@
+#include "archive-mover.hpp"
+#include "td/actor/MultiPromise.h"
+#include "validator/fabric.h"
+
+namespace ton {
+
+namespace validator {
+
+void ArchiveFileMover::start_up() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+    td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R));
+  });
+  td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, block_id_, std::move(P));
+}
+
+void ArchiveFileMover::got_block_handle0(td::Result<BlockHandle> R) {
+  if (R.is_ok()) {
+    handle_ = R.move_as_ok();
+    CHECK(handle_->moved_to_archive());
+    CHECK(handle_->handle_moved_to_archive());
+    finish_query();
+    return;
+  }
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+    td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R));
+  });
+  td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read_handle, block_id_, std::move(P));
+}
+
+void ArchiveFileMover::got_block_handle1(td::Result<BlockHandle> R) {
+  if (R.is_ok()) {
+    handle_ = R.move_as_ok();
+    got_block_handle();
+    return;
+  }
+
+  if (R.error().code() != ErrorCode::notready) {
+    abort_query(R.move_as_error());
+    return;
+  }
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+    td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R));
+  });
+  td::actor::send_closure(block_db_, &BlockDb::get_block_handle, std::move(P));
+}
+
+void ArchiveFileMover::got_block_handle2(td::Result<BlockHandle> R) {
+  if (R.is_ok()) {
+    handle_ = R.move_as_ok();
+    got_block_handle();
+    return;
+  }
+
+  if (R.error().code() != ErrorCode::notready) {
+    abort_query(R.move_as_error());
+    return;
+  }
+
+  finish_query();
+}
+
+void ArchiveFileMover::got_block_handle() {
+  if (!handle_->is_applied()) {
+    finish_query();
+    return;
+  }
+  if (handle_->id().seqno() == 0) {
+    processed_all_children();
+    return;
+  }
+
+  CHECK(handle_->inited_prev());
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child);
+    }
+  });
+
+  td::actor::create_actor<ArchiveFileMover>("mover", handle_->one_prev(left_), mode_, block_db_, file_db_,
+                                            old_archive_db_, old_archive_manager_, archive_manager_, std::move(P))
+      .release();
+}
+
+void ArchiveFileMover::processed_child() {
+  if (!left_ || !handle_->merge_before()) {
+    processed_all_children();
+    return;
+  }
+  left_ = false;
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child);
+    }
+  });
+
+  td::actor::create_actor<ArchiveFileMover>("mover", handle_->one_prev(left_), mode_, block_db_, file_db_,
+                                            old_archive_db_, old_archive_manager_, archive_manager_, std::move(P))
+      .release();
+}
+
+void ArchiveFileMover::processed_all_children() {
+  if (!handle_->received()) {
+    got_block_data(td::BufferSlice{});
+  } else {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_data, std::move(R));
+    });
+
+    if (handle_->moved_to_archive()) {
+      CHECK(handle_->inited_unix_time());
+      td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(),
+                              handle_->is_key_block(), FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P));
+    } else {
+      td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file,
+                              FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P));
+    }
+  }
+}
+
+void ArchiveFileMover::got_block_data(td::Result<td::BufferSlice> R) {
+  if (R.is_error()) {
+    if (R.error().code() != ErrorCode::notready) {
+      abort_query(R.move_as_error());
+      return;
+    }
+  } else {
+    data_ = R.move_as_ok();
+  }
+  if (!handle_->inited_proof()) {
+    got_block_proof(td::BufferSlice{});
+  } else {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof, std::move(R));
+    });
+
+    if (handle_->moved_to_archive()) {
+      CHECK(handle_->inited_unix_time());
+      td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(),
+                              handle_->is_key_block(), FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P));
+    } else {
+      td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file,
+                              FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P));
+    }
+  }
+}
+
+void ArchiveFileMover::got_block_proof(td::Result<td::BufferSlice> R) {
+  if (R.is_error()) {
+    if (R.error().code() != ErrorCode::notready) {
+      abort_query(R.move_as_error());
+      return;
+    }
+  } else {
+    proof_ = R.move_as_ok();
+  }
+  if (!handle_->inited_proof_link()) {
+    got_block_proof_link(td::BufferSlice{});
+  } else {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof_link, std::move(R));
+    });
+
+    if (handle_->moved_to_archive()) {
+      CHECK(handle_->inited_unix_time());
+      td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(),
+                              handle_->is_key_block(), FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P));
+    } else {
+      td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file,
+                              FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P));
+    }
+  }
+}
+
+void ArchiveFileMover::got_block_proof_link(td::Result<td::BufferSlice> R) {
+  if (R.is_error()) {
+    if (R.error().code() != ErrorCode::notready) {
+      abort_query(R.move_as_error());
+      return;
+    }
+  } else {
+    proof_link_ = R.move_as_ok();
+  }
+
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveFileMover::written_data);
+  });
+
+  if (data_.size() > 0) {
+    td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Block{block_id_},
+                            std::move(data_), ig.get_promise());
+  }
+  if (proof_.size() > 0) {
+    td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Proof{block_id_},
+                            std::move(proof_), ig.get_promise());
+  }
+  if (proof_link_.size() > 0) {
+    td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::ProofLink{block_id_},
+                            std::move(proof_link_), ig.get_promise());
+  }
+}
+
+void ArchiveFileMover::written_data() {
+  handle_->set_moved_to_archive();
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveFileMover::written_handle);
+  });
+  td::actor::send_closure(archive_manager_, &ArchiveManager::add_handle, handle_, std::move(P));
+}
+
+void ArchiveFileMover::written_handle() {
+  CHECK(handle_->handle_moved_to_archive());
+  finish_query();
+}
+
+void ArchiveFileMover::abort_query(td::Status error) {
+  if (promise_) {
+    promise_.set_error(std::move(error));
+  }
+  stop();
+}
+
+void ArchiveFileMover::finish_query() {
+  if (promise_) {
+    promise_.set_value(td::Unit());
+  }
+  stop();
+}
+
+void ArchiveKeyBlockMover::start_up() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    if (R.is_ok()) {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::skip_block_proof, R.move_as_ok());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof0);
+    }
+  });
+  if (proof_link_) {
+    td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::ProofLink{block_id_},
+                            std::move(P));
+  } else {
+    td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::Proof{block_id_}, std::move(P));
+  }
+}
+
+void ArchiveKeyBlockMover::failed_to_get_proof0() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    if (R.is_ok()) {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof1);
+    }
+  });
+  if (proof_link_) {
+    td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::ProofLink{block_id_},
+                            std::move(P));
+  } else {
+    td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::Proof{block_id_}, std::move(P));
+  }
+}
+
+void ArchiveKeyBlockMover::failed_to_get_proof1() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    if (R.is_ok()) {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof2);
+    }
+  });
+  if (proof_link_) {
+    td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P));
+  } else {
+    td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P));
+  }
+}
+
+void ArchiveKeyBlockMover::failed_to_get_proof2() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    if (R.is_ok()) {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof3);
+    }
+  });
+  if (proof_link_) {
+    td::actor::send_closure(file_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P));
+  } else {
+    td::actor::send_closure(file_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P));
+  }
+}
+
+void ArchiveKeyBlockMover::failed_to_get_proof3() {
+  if (proof_link_) {
+    written_data();
+  } else {
+    proof_link_ = true;
+    start_up();
+  }
+}
+
+void ArchiveKeyBlockMover::got_block_proof(td::BufferSlice data) {
+  data_ = std::move(data);
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::written_data);
+  });
+
+  if (proof_link_) {
+    auto p = create_proof_link(block_id_, data_.clone()).move_as_ok();
+    auto h = p->get_basic_header_info().move_as_ok();
+    td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime,
+                            fileref::ProofLink{block_id_}, std::move(data_), std::move(P));
+  } else {
+    auto p = create_proof(block_id_, data_.clone()).move_as_ok();
+    auto h = p->get_basic_header_info().move_as_ok();
+    td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime, fileref::Proof{block_id_},
+                            std::move(data_), std::move(P));
+  }
+}
+
+void ArchiveKeyBlockMover::skip_block_proof(td::BufferSlice data) {
+  data_ = std::move(data);
+  written_data();
+}
+
+void ArchiveKeyBlockMover::written_data() {
+  td::Ref<ProofLink> proof_link;
+  if (proof_link_) {
+    auto p = create_proof_link(block_id_, data_.clone()).move_as_ok();
+    proof_link = std::move(p);
+  } else {
+    auto p = create_proof(block_id_, data_.clone()).move_as_ok();
+    proof_link = std::move(p);
+  }
+  auto ts = proof_link->get_basic_header_info().move_as_ok().utime;
+  auto te = ValidatorManager::persistent_state_ttl(ts);
+  if (te < td::Clocks::system()) {
+    finish_query();
+    return;
+  }
+}
+
+void ArchiveKeyBlockMover::abort_query(td::Status error) {
+  if (promise_) {
+    promise_.set_error(std::move(error));
+  }
+  stop();
+}
+
+void ArchiveKeyBlockMover::finish_query() {
+  if (promise_) {
+    promise_.set_value(td::Unit());
+  }
+  stop();
+}
+
+void ArchiveMover::start_up() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveMover::moved_blocks);
+    }
+  });
+  td::actor::create_actor<ArchiveFileMover>("fmover", masterchain_block_id_, block_db_.get(), file_db_.get(),
+                                            old_archive_db_.get(), old_archive_manager_.get(), archive_manager_.get(),
+                                            std::move(P))
+      .release();
+}
+
+void ArchiveMover::moved_blocks() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveMover::got_handle, R.move_as_ok());
+  });
+  td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, masterchain_block_id_, std::move(P));
+}
+
+void ArchiveMover::got_handle(BlockHandle handle) {
+  handle_ = std::move(handle);
+  CHECK(handle_->is_applied());
+  CHECK(handle_->inited_state_boc());
+  CHECK(!handle_->deleted_state_boc());
+  auto P = td::PromiseCreator::lambda(
+      [handle = handle_, SelfId = actor_id(this)](td::Result<td::Ref<vm::DataCell>> R) mutable {
+        R.ensure();
+        auto S = create_shard_state(handle->id(), R.move_as_ok());
+        S.ensure();
+        td::actor::send_closure(SelfId, &ArchiveMover::got_state, td::Ref<MasterchainState>{S.move_as_ok()});
+      });
+  td::actor::send_closure(cell_db_, &CellDb::load_cell, handle_->state(), std::move(P));
+}
+
+void ArchiveMover::got_state(td::Ref<MasterchainState> state) {
+  state_ = std::move(state);
+
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks);
+    }
+  });
+
+  auto k = state_->prev_key_block_id(std::numeric_limits<BlockSeqno>::max());
+  while (k.is_valid() && k.seqno() > 0) {
+    td::actor::create_actor<ArchiveKeyBlockMover>("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(),
+                                                  old_archive_manager_.get(), archive_manager_.get(), ig.get_promise())
+        .release();
+    k = state_->prev_key_block_id(k.seqno());
+  }
+}
+
+void ArchiveMover::moved_key_blocks() {
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks);
+    }
+  });
+
+  auto k = state_->prev_key_block_id(std::numeric_limits<BlockSeqno>::max());
+  while (k.is_valid() && k.seqno() > 0) {
+    td::actor::create_actor<ArchiveKeyBlockMover>("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(),
+                                                  old_archive_manager_.get(), archive_manager_.get(), ig.get_promise())
+        .release();
+    k = state_->prev_key_block_id(k.seqno());
+  }
+}
+
+void ArchiveMover::run() {
+  if (to_move_.empty() && to_check_.empty()) {
+    completed();
+    return;
+  }
+  if (!to_check_.empty()) {
+    auto B = to_check_.back();
+    CHECK(to_check_set_.count(B) == 1);
+
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+      td::actor::send_closure(SelfId, &ArchiveMover::got_to_check_handle, std::move(R));
+    });
+    td::actor::send_closure(block_db_, &BlockDb::get_block_handle, B, std::move(P));
+    return;
+  }
+  CHECK(!to_move_.empty());
+}
+
+void ArchiveMover::got_to_check_handle(td::Result<BlockHandle> R) {
+  if (R.is_error()) {
+    CHECK(R.error().code() == ErrorCode::notready);
+    run();
+    return;
+  }
+  auto handle = R.move_as_ok();
+}
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/archive-mover.hpp

@@ -0,0 +1,148 @@
+#pragma once
+
+#include "td/actor/actor.h"
+#include "filedb.hpp"
+#include "blockdb.hpp"
+#include "statedb.hpp"
+#include "celldb.hpp"
+#include "archive-db.hpp"
+#include "archive-manager.hpp"
+
+#include <list>
+#include <set>
+
+namespace ton {
+
+namespace validator {
+
+class ArchiveFileMover : public td::actor::Actor {
+ public:
+  ArchiveFileMover(BlockIdExt block_id, td::actor::ActorId<BlockDb> block_db, td::actor::ActorId<FileDb> file_db,
+                   td::actor::ActorId<FileDb> old_archive_db, td::actor::ActorId<OldArchiveManager> old_archive_manager,
+                   td::actor::ActorId<ArchiveManager> archive_manager, td::Promise<td::Unit> promise)
+      : block_id_(block_id)
+      , block_db_(block_db)
+      , file_db_(file_db)
+      , old_archive_db_(old_archive_db)
+      , old_archive_manager_(old_archive_manager)
+      , archive_manager_(archive_manager)
+      , promise_(std::move(promise)) {
+  }
+  void start_up() override;
+  void got_block_handle0(td::Result<BlockHandle> R);
+  void got_block_handle1(td::Result<BlockHandle> R);
+  void got_block_handle2(td::Result<BlockHandle> R);
+  void got_block_handle();
+
+  void processed_child();
+  void processed_all_children();
+
+  void got_block_data(td::Result<td::BufferSlice> R);
+  void got_block_proof(td::Result<td::BufferSlice> R);
+  void got_block_proof_link(td::Result<td::BufferSlice> R);
+
+  void written_data();
+  void written_handle();
+
+  void abort_query(td::Status error);
+  void finish_query();
+
+ private:
+  BlockIdExt block_id_;
+  BlockHandle handle_;
+  td::BufferSlice data_;
+  td::BufferSlice proof_;
+  td::BufferSlice proof_link_;
+  bool left_ = true;
+
+  td::actor::ActorId<BlockDb> block_db_;
+  td::actor::ActorId<FileDb> file_db_;
+  td::actor::ActorId<FileDb> old_archive_db_;
+  td::actor::ActorId<OldArchiveManager> old_archive_manager_;
+  td::actor::ActorId<ArchiveManager> archive_manager_;
+
+  td::Promise<td::Unit> promise_;
+};
+
+class ArchiveKeyBlockMover : public td::actor::Actor {
+ public:
+  ArchiveKeyBlockMover(BlockIdExt block_id, td::actor::ActorId<BlockDb> block_db, td::actor::ActorId<FileDb> file_db,
+                       td::actor::ActorId<FileDb> old_archive_db,
+                       td::actor::ActorId<OldArchiveManager> old_archive_manager,
+                       td::actor::ActorId<ArchiveManager> archive_manager, td::Promise<td::Unit> promise)
+      : block_id_(block_id)
+      , block_db_(block_db)
+      , file_db_(file_db)
+      , old_archive_db_(old_archive_db)
+      , old_archive_manager_(old_archive_manager)
+      , archive_manager_(archive_manager)
+      , promise_(std::move(promise)) {
+  }
+
+  void start_up() override;
+  void failed_to_get_proof0();
+  void failed_to_get_proof1();
+  void failed_to_get_proof2();
+  void failed_to_get_proof3();
+  void got_block_proof(td::BufferSlice data);
+  void skip_block_proof(td::BufferSlice data);
+
+  void written_data();
+
+  void abort_query(td::Status error);
+  void finish_query();
+
+ private:
+  BlockIdExt block_id_;
+  td::BufferSlice data_;
+  bool proof_link_ = false;
+
+  td::actor::ActorId<BlockDb> block_db_;
+  td::actor::ActorId<FileDb> file_db_;
+  td::actor::ActorId<FileDb> old_archive_db_;
+  td::actor::ActorId<OldArchiveManager> old_archive_manager_;
+  td::actor::ActorId<ArchiveManager> archive_manager_;
+
+  td::Promise<td::Unit> promise_;
+};
+
+class ArchiveMover : public td::actor::Actor {
+ public:
+  ArchiveMover(std::string db_root, BlockIdExt masterchain_block_id, BlockIdExt shard_block_id,
+               BlockIdExt key_block_id);
+
+  void start_up() override;
+  void moved_blocks();
+  void got_handle(BlockHandle handle);
+  void got_state(td::Ref<MasterchainState> state);
+  void moved_key_blocks();
+  void run();
+  void completed();
+  void add_to_move(BlockIdExt block_id);
+  void add_to_check(BlockIdExt block_id);
+
+  void got_to_check_handle(td::Result<BlockHandle> R);
+
+  void abort_query(td::Status error);
+  void finish_query();
+
+ private:
+  std::string db_root_;
+  BlockHandle handle_;
+  td::Ref<MasterchainState> state_;
+
+  td::actor::ActorOwn<BlockDb> block_db_;
+  td::actor::ActorOwn<FileDb> file_db_;
+  td::actor::ActorOwn<FileDb> old_archive_db_;
+  td::actor::ActorOwn<OldArchiveManager> old_archive_manager_;
+  td::actor::ActorOwn<ArchiveManager> archive_manager_;
+  td::actor::ActorOwn<CellDb> cell_db_;
+
+  BlockIdExt masterchain_block_id_;
+  BlockIdExt shard_block_id_;
+  BlockIdExt key_block_id_;
+};
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/archive-slice.cpp

@@ -0,0 +1,470 @@
+#include "archive-slice.hpp"
+
+#include "td/actor/MultiPromise.h"
+#include "validator/fabric.h"
+#include "td/db/RocksDb.h"
+#include "ton/ton-io.hpp"
+#include "td/utils/port/path.h"
+#include "common/delay.h"
+#include "files-async.hpp"
+
+namespace ton {
+
+namespace validator {
+
+void PackageWriter::append(std::string filename, td::BufferSlice data,
+                           td::Promise<std::pair<td::uint64, td::uint64>> promise) {
+  auto offset = package_->append(std::move(filename), std::move(data), !async_mode_);
+  auto size = package_->size();
+
+  promise.set_value(std::pair<td::uint64, td::uint64>{offset, size});
+}
+
+class PackageReader : public td::actor::Actor {
+ public:
+  PackageReader(std::shared_ptr<Package> package, td::uint64 offset,
+                td::Promise<std::pair<std::string, td::BufferSlice>> promise)
+      : package_(std::move(package)), offset_(offset), promise_(std::move(promise)) {
+  }
+  void start_up() {
+    promise_.set_result(package_->read(offset_));
+  }
+
+ private:
+  std::shared_ptr<Package> package_;
+  td::uint64 offset_;
+  td::Promise<std::pair<std::string, td::BufferSlice>> promise_;
+};
+
+void ArchiveSlice::add_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  if (handle->id().seqno() == 0) {
+    update_handle(std::move(handle), std::move(promise));
+    return;
+  }
+  CHECK(!key_blocks_only_);
+  CHECK(!temp_);
+  CHECK(handle->inited_unix_time());
+  CHECK(handle->inited_logical_time());
+
+  auto key = get_db_key_lt_desc(handle->id().shard_full());
+
+  std::string value;
+  auto R = kv_->get(key.as_slice(), value);
+  R.ensure();
+  tl_object_ptr<ton_api::db_lt_desc_value> v;
+  bool add_shard = false;
+  if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+    auto F = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
+    F.ensure();
+    v = F.move_as_ok();
+  } else {
+    v = create_tl_object<ton_api::db_lt_desc_value>(1, 1, 0, 0, 0);
+    add_shard = true;
+  }
+  if (handle->id().seqno() <= static_cast<td::uint32>(v->last_seqno_) ||
+      handle->logical_time() <= static_cast<LogicalTime>(v->last_lt_) ||
+      handle->unix_time() <= static_cast<UnixTime>(v->last_ts_)) {
+    update_handle(std::move(handle), std::move(promise));
+    return;
+  }
+  auto db_value = create_serialize_tl_object<ton_api::db_lt_el_value>(create_tl_block_id(handle->id()),
+                                                                      handle->logical_time(), handle->unix_time());
+  auto db_key = get_db_key_lt_el(handle->id().shard_full(), v->last_idx_++);
+  auto status_key = create_serialize_tl_object<ton_api::db_lt_status_key>();
+  v->last_seqno_ = handle->id().seqno();
+  v->last_lt_ = handle->logical_time();
+  v->last_ts_ = handle->unix_time();
+
+  td::uint32 idx = 0;
+  if (add_shard) {
+    auto G = kv_->get(status_key.as_slice(), value);
+    G.ensure();
+    if (G.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+      idx = 0;
+    } else {
+      auto F = fetch_tl_object<ton_api::db_lt_status_value>(value, true);
+      F.ensure();
+      auto f = F.move_as_ok();
+      idx = f->total_shards_;
+    }
+  }
+
+  auto version = handle->version();
+
+  begin_transaction();
+  kv_->set(key, serialize_tl_object(v, true)).ensure();
+  kv_->set(db_key, db_value.as_slice()).ensure();
+  if (add_shard) {
+    auto shard_key = create_serialize_tl_object<ton_api::db_lt_shard_key>(idx);
+    auto shard_value =
+        create_serialize_tl_object<ton_api::db_lt_shard_value>(handle->id().id.workchain, handle->id().id.shard);
+    kv_->set(status_key.as_slice(), create_serialize_tl_object<ton_api::db_lt_status_value>(idx + 1)).ensure();
+    kv_->set(shard_key.as_slice(), shard_value.as_slice()).ensure();
+  }
+  kv_->set(get_db_key_block_info(handle->id()), handle->serialize().as_slice()).ensure();
+  commit_transaction();
+
+  handle->flushed_upto(version);
+  handle->set_handle_moved_to_archive();
+
+  if (handle->need_flush()) {
+    update_handle(std::move(handle), std::move(promise));
+  } else {
+    promise.set_value(td::Unit());
+  }
+}
+
+void ArchiveSlice::update_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  if (!handle->need_flush() && (temp_ || handle->handle_moved_to_archive())) {
+    promise.set_value(td::Unit());
+    return;
+  }
+  CHECK(!key_blocks_only_);
+
+  begin_transaction();
+  do {
+    auto version = handle->version();
+    kv_->set(get_db_key_block_info(handle->id()), handle->serialize().as_slice()).ensure();
+    handle->flushed_upto(version);
+  } while (handle->need_flush());
+  commit_transaction();
+  if (!temp_) {
+    handle->set_handle_moved_to_archive();
+  }
+
+  promise.set_value(td::Unit());
+}
+
+void ArchiveSlice::add_file(FileReference ref_id, td::BufferSlice data, td::Promise<td::Unit> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  std::string value;
+  auto R = kv_->get(ref_id.hash().to_hex(), value);
+  R.ensure();
+  if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+    promise.set_value(td::Unit());
+    return;
+  }
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id, promise = std::move(promise)](
+                                          td::Result<std::pair<td::uint64, td::uint64>> R) mutable {
+    if (R.is_error()) {
+      promise.set_error(R.move_as_error());
+      return;
+    }
+    auto v = R.move_as_ok();
+    td::actor::send_closure(SelfId, &ArchiveSlice::add_file_cont, std::move(ref_id), v.first, v.second,
+                            std::move(promise));
+  });
+
+  td::actor::send_closure(writer_, &PackageWriter::append, ref_id.filename(), std::move(data), std::move(P));
+}
+
+void ArchiveSlice::add_file_cont(FileReference ref_id, td::uint64 offset, td::uint64 size,
+                                 td::Promise<td::Unit> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  begin_transaction();
+  kv_->set("status", td::to_string(size)).ensure();
+  kv_->set(ref_id.hash().to_hex(), td::to_string(offset)).ensure();
+  commit_transaction();
+  promise.set_value(td::Unit());
+}
+
+void ArchiveSlice::get_handle(BlockIdExt block_id, td::Promise<BlockHandle> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  CHECK(!key_blocks_only_);
+  std::string value;
+  auto R = kv_->get(get_db_key_block_info(block_id), value);
+  R.ensure();
+  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "handle not in archive slice"));
+    return;
+  }
+  auto E = create_block_handle(td::BufferSlice{value});
+  E.ensure();
+  auto handle = E.move_as_ok();
+  if (!temp_) {
+    handle->set_handle_moved_to_archive();
+  }
+  promise.set_value(std::move(handle));
+}
+
+void ArchiveSlice::get_file(FileReference ref_id, td::Promise<td::BufferSlice> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  std::string value;
+  auto R = kv_->get(ref_id.hash().to_hex(), value);
+  R.ensure();
+  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "file not in archive slice"));
+    return;
+  }
+  auto offset = td::to_integer<td::uint64>(value);
+  auto P = td::PromiseCreator::lambda(
+      [promise = std::move(promise)](td::Result<std::pair<std::string, td::BufferSlice>> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          promise.set_value(std::move(R.move_as_ok().second));
+        }
+      });
+  td::actor::create_actor<PackageReader>("reader", package_, offset, std::move(P)).release();
+}
+
+void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id,
+                                    std::function<td::int32(ton_api::db_lt_desc_value &)> compare_desc,
+                                    std::function<td::int32(ton_api::db_lt_el_value &)> compare, bool exact,
+                                    td::Promise<BlockHandle> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  bool f = false;
+  BlockIdExt block_id;
+  td::uint32 ls = 0;
+  for (td::uint32 len = 0; len <= 60; len++) {
+    auto s = shard_prefix(account_id, len);
+    auto key = get_db_key_lt_desc(s);
+    std::string value;
+    auto F = kv_->get(key, value);
+    F.ensure();
+    if (F.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+      if (!f) {
+        continue;
+      } else {
+        break;
+      }
+    }
+    f = true;
+    auto G = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
+    G.ensure();
+    auto g = G.move_as_ok();
+    if (compare_desc(*g.get()) > 0) {
+      continue;
+    }
+    td::uint32 l = g->first_idx_ - 1;
+    BlockIdExt lseq;
+    td::uint32 r = g->last_idx_;
+    BlockIdExt rseq;
+    while (r - l > 1) {
+      auto x = (r + l) / 2;
+      auto db_key = get_db_key_lt_el(s, x);
+      F = kv_->get(db_key, value);
+      F.ensure();
+      CHECK(F.move_as_ok() == td::KeyValue::GetStatus::Ok);
+      auto E = fetch_tl_object<ton_api::db_lt_el_value>(td::BufferSlice{value}, true);
+      E.ensure();
+      auto e = E.move_as_ok();
+      int cmp_val = compare(*e.get());
+
+      if (cmp_val < 0) {
+        rseq = create_block_id(e->id_);
+        r = x;
+      } else if (cmp_val > 0) {
+        lseq = create_block_id(e->id_);
+        l = x;
+      } else {
+        get_handle(create_block_id(e->id_), std::move(promise));
+        return;
+      }
+    }
+    if (rseq.is_valid()) {
+      if (!block_id.is_valid()) {
+        block_id = rseq;
+      } else if (block_id.id.seqno > rseq.id.seqno) {
+        block_id = rseq;
+      }
+    }
+    if (lseq.is_valid()) {
+      if (ls < lseq.id.seqno) {
+        ls = lseq.id.seqno;
+      }
+    }
+    if (block_id.is_valid() && ls + 1 == block_id.id.seqno) {
+      if (!exact) {
+        get_handle(block_id, std::move(promise));
+      } else {
+        promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found"));
+      }
+      return;
+    }
+  }
+  if (!exact && block_id.is_valid()) {
+    get_handle(block_id, std::move(promise));
+  } else {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found"));
+  }
+}
+
+void ArchiveSlice::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockHandle> promise) {
+  return get_block_common(
+      account_id,
+      [lt](ton_api::db_lt_desc_value &w) {
+        return lt > static_cast<LogicalTime>(w.last_lt_) ? 1 : lt == static_cast<LogicalTime>(w.last_lt_) ? 0 : -1;
+      },
+      [lt](ton_api::db_lt_el_value &w) {
+        return lt > static_cast<LogicalTime>(w.lt_) ? 1 : lt == static_cast<LogicalTime>(w.lt_) ? 0 : -1;
+      },
+      false, std::move(promise));
+}
+
+void ArchiveSlice::get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno,
+                                      td::Promise<BlockHandle> promise) {
+  return get_block_common(
+      account_id,
+      [seqno](ton_api::db_lt_desc_value &w) {
+        return seqno > static_cast<BlockSeqno>(w.last_seqno_)
+                   ? 1
+                   : seqno == static_cast<BlockSeqno>(w.last_seqno_) ? 0 : -1;
+      },
+      [seqno](ton_api::db_lt_el_value &w) {
+        return seqno > static_cast<BlockSeqno>(w.id_->seqno_)
+                   ? 1
+                   : seqno == static_cast<BlockSeqno>(w.id_->seqno_) ? 0 : -1;
+      },
+      true, std::move(promise));
+}
+
+void ArchiveSlice::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts,
+                                          td::Promise<BlockHandle> promise) {
+  return get_block_common(
+      account_id,
+      [ts](ton_api::db_lt_desc_value &w) {
+        return ts > static_cast<UnixTime>(w.last_ts_) ? 1 : ts == static_cast<UnixTime>(w.last_ts_) ? 0 : -1;
+      },
+      [ts](ton_api::db_lt_el_value &w) {
+        return ts > static_cast<UnixTime>(w.ts_) ? 1 : ts == static_cast<UnixTime>(w.ts_) ? 0 : -1;
+      },
+      false, std::move(promise));
+}
+
+td::BufferSlice ArchiveSlice::get_db_key_lt_desc(ShardIdFull shard) {
+  return create_serialize_tl_object<ton_api::db_lt_desc_key>(shard.workchain, shard.shard);
+}
+
+td::BufferSlice ArchiveSlice::get_db_key_lt_el(ShardIdFull shard, td::uint32 idx) {
+  return create_serialize_tl_object<ton_api::db_lt_el_key>(shard.workchain, shard.shard, idx);
+}
+
+td::BufferSlice ArchiveSlice::get_db_key_block_info(BlockIdExt block_id) {
+  return create_serialize_tl_object<ton_api::db_blockdb_key_value>(create_tl_block_id(block_id));
+}
+
+void ArchiveSlice::get_slice(td::uint64 offset, td::uint32 limit, td::Promise<td::BufferSlice> promise) {
+  td::actor::create_actor<db::ReadFile>("readfile", prefix_ + ".pack", offset, limit, 0, std::move(promise)).release();
+}
+
+void ArchiveSlice::start_up() {
+  auto R = Package::open(prefix_ + ".pack", false, true);
+  if (R.is_error()) {
+    LOG(FATAL) << "failed to open/create archive '" << prefix_ << ".pack"
+               << "': " << R.move_as_error();
+    return;
+  }
+  package_ = std::make_shared<Package>(R.move_as_ok());
+  kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(prefix_ + ".index").move_as_ok());
+
+  std::string value;
+  auto R2 = kv_->get("status", value);
+  R2.ensure();
+
+  if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+    auto len = td::to_integer<td::uint64>(value);
+    package_->truncate(len);
+  } else {
+    package_->truncate(0);
+  }
+
+  writer_ = td::actor::create_actor<PackageWriter>("writer", package_);
+}
+
+void ArchiveSlice::begin_transaction() {
+  if (!async_mode_ || !huge_transaction_started_) {
+    kv_->begin_transaction().ensure();
+    if (async_mode_) {
+      huge_transaction_started_ = true;
+    }
+  }
+}
+
+void ArchiveSlice::commit_transaction() {
+  if (!async_mode_ || huge_transaction_size_++ >= 100) {
+    kv_->commit_transaction().ensure();
+    if (async_mode_) {
+      huge_transaction_size_ = 0;
+      huge_transaction_started_ = false;
+    }
+  }
+}
+
+void ArchiveSlice::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
+  async_mode_ = mode;
+  if (!async_mode_ && huge_transaction_started_) {
+    kv_->commit_transaction().ensure();
+    huge_transaction_size_ = 0;
+    huge_transaction_started_ = false;
+  }
+
+  td::actor::send_closure(writer_, &PackageWriter::set_async_mode, mode, std::move(promise));
+}
+
+ArchiveSlice::ArchiveSlice(bool key_blocks_only, bool temp, std::string prefix)
+    : key_blocks_only_(key_blocks_only), temp_(temp), prefix_(std::move(prefix)) {
+}
+
+namespace {
+
+void destroy_db(std::string name, td::uint32 attempt, td::Promise<td::Unit> promise) {
+  auto S = td::RocksDb::destroy(name);
+  if (S.is_ok()) {
+    promise.set_value(td::Unit());
+    return;
+  }
+  if (S.is_error() && attempt > 0 && attempt % 64 == 0) {
+    LOG(ERROR) << "failed to destroy index " << name << ": " << S;
+  } else {
+    LOG(DEBUG) << "failed to destroy index " << name << ": " << S;
+  }
+  delay_action(
+      [name, attempt, promise = std::move(promise)]() mutable { destroy_db(name, attempt, std::move(promise)); },
+      td::Timestamp::in(1.0));
+}
+}  // namespace
+
+void ArchiveSlice::destroy(td::Promise<td::Unit> promise) {
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise(std::move(promise));
+  destroyed_ = true;
+
+  writer_.reset();
+  package_ = nullptr;
+  kv_ = nullptr;
+
+  td::unlink(prefix_ + ".pack").ensure();
+
+  delay_action([name = prefix_ + ".index", attempt = 0,
+                promise = ig.get_promise()]() mutable { destroy_db(name, attempt, std::move(promise)); },
+               td::Timestamp::in(0.0));
+}
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/archive-slice.hpp

@@ -0,0 +1,84 @@
+#pragma once
+
+#include "validator/interfaces/db.h"
+#include "package.hpp"
+#include "fileref.hpp"
+
+namespace ton {
+
+namespace validator {
+
+class PackageWriter : public td::actor::Actor {
+ public:
+  PackageWriter(std::shared_ptr<Package> package) : package_(std::move(package)) {
+  }
+
+  void append(std::string filename, td::BufferSlice data, td::Promise<std::pair<td::uint64, td::uint64>> promise);
+  void set_async_mode(bool mode, td::Promise<td::Unit> promise) {
+    async_mode_ = mode;
+    if (!async_mode_) {
+      package_->sync();
+    }
+    promise.set_value(td::Unit());
+  }
+
+ private:
+  std::shared_ptr<Package> package_;
+  bool async_mode_ = false;
+};
+
+class ArchiveSlice : public td::actor::Actor {
+ public:
+  ArchiveSlice(bool key_blocks_only, bool temp, std::string prefix);
+
+  void add_handle(BlockHandle handle, td::Promise<td::Unit> promise);
+  void update_handle(BlockHandle handle, td::Promise<td::Unit> promise);
+  void add_file(FileReference ref_id, td::BufferSlice data, td::Promise<td::Unit> promise);
+  void get_handle(BlockIdExt block_id, td::Promise<BlockHandle> promise);
+  void get_file(FileReference ref_id, td::Promise<td::BufferSlice> promise);
+
+  /* from LTDB */
+  void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise<BlockHandle> promise);
+  void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockHandle> promise);
+  void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise<BlockHandle> promise);
+  void get_block_common(AccountIdPrefixFull account_id,
+                        std::function<td::int32(ton_api::db_lt_desc_value &)> compare_desc,
+                        std::function<td::int32(ton_api::db_lt_el_value &)> compare, bool exact,
+                        td::Promise<BlockHandle> promise);
+
+  void get_slice(td::uint64 offset, td::uint32 limit, td::Promise<td::BufferSlice> promise);
+
+  void start_up() override;
+  void destroy(td::Promise<td::Unit> promise);
+
+  void begin_transaction();
+  void commit_transaction();
+  void set_async_mode(bool mode, td::Promise<td::Unit> promise);
+
+ private:
+  void written_data(BlockHandle handle, td::Promise<td::Unit> promise);
+  void add_file_cont(FileReference ref_id, td::uint64 offset, td::uint64 size, td::Promise<td::Unit> promise);
+
+  /* ltdb */
+  td::BufferSlice get_db_key_lt_desc(ShardIdFull shard);
+  td::BufferSlice get_db_key_lt_el(ShardIdFull shard, td::uint32 idx);
+  td::BufferSlice get_db_key_block_info(BlockIdExt block_id);
+  td::BufferSlice get_lt_from_db(ShardIdFull shard, td::uint32 idx);
+
+  bool key_blocks_only_;
+  bool temp_;
+
+  bool destroyed_ = false;
+  bool async_mode_ = false;
+  bool huge_transaction_started_ = false;
+  td::uint32 huge_transaction_size_ = 0;
+
+  std::string prefix_;
+  std::shared_ptr<Package> package_;
+  std::shared_ptr<td::KeyValue> kv_;
+  td::actor::ActorOwn<PackageWriter> writer_;
+};
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/archiver.cpp

@@ -24,41 +24,30 @@ namespace ton {
 
 namespace validator {
 
-BlockArchiver::BlockArchiver(BlockIdExt block_id, td::actor::ActorId<RootDb> root_db,
-                             td::actor::ActorId<FileDb> file_db, td::actor::ActorId<FileDb> archive_db,
-                             td::actor::ActorId<ArchiveManager> archive, td::Promise<td::Unit> promise)
-    : block_id_(block_id)
-    , root_db_(root_db)
-    , file_db_(file_db)
-    , archive_db_(archive_db)
-    , archive_(archive)
-    , promise_(std::move(promise)) {
+BlockArchiver::BlockArchiver(BlockHandle handle, td::actor::ActorId<ArchiveManager> archive_db,
+                             td::Promise<td::Unit> promise)
+    : handle_(std::move(handle)), archive_(archive_db), promise_(std::move(promise)) {
 }
 
 void BlockArchiver::start_up() {
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
-    R.ensure();
-    td::actor::send_closure(SelfId, &BlockArchiver::got_block_handle, R.move_as_ok());
-  });
-  td::actor::send_closure(root_db_, &RootDb::get_block_handle_external, block_id_, false, std::move(P));
+  if (handle_->handle_moved_to_archive()) {
+    moved_handle();
+  } else {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+      R.ensure();
+      td::actor::send_closure(SelfId, &BlockArchiver::moved_handle);
+    });
+    td::actor::send_closure(archive_, &ArchiveManager::add_handle, handle_, std::move(P));
+  }
 }
 
-void BlockArchiver::got_block_handle(BlockHandle handle) {
-  handle_ = std::move(handle);
+void BlockArchiver::moved_handle() {
+  CHECK(handle_->handle_moved_to_archive());
   if (handle_->moved_to_archive()) {
     finish_query();
     return;
   }
 
-  if (!handle_->is_applied() && !handle_->is_archived() &&
-      (!handle_->inited_is_key_block() || !handle_->is_key_block())) {
-    // no need for this block
-    // probably this block not in final chain
-    // this will eventually delete all associated data
-    written_block_data();
-    return;
-  }
-
   if (!handle_->inited_proof()) {
     written_proof();
     return;
@@ -69,11 +58,7 @@ void BlockArchiver::got_block_handle(BlockHandle handle) {
     td::actor::send_closure(SelfId, &BlockArchiver::got_proof, R.move_as_ok());
   });
 
-  if (handle_->moved_to_storage()) {
-    td::actor::send_closure(archive_db_, &FileDb::load_file, FileDb::RefId{fileref::Proof{block_id_}}, std::move(P));
-  } else {
-    td::actor::send_closure(file_db_, &FileDb::load_file, FileDb::RefId{fileref::Proof{block_id_}}, std::move(P));
-  }
+  td::actor::send_closure(archive_, &ArchiveManager::get_temp_file_short, fileref::Proof{handle_->id()}, std::move(P));
 }
 
 void BlockArchiver::got_proof(td::BufferSlice data) {
@@ -81,8 +66,8 @@ void BlockArchiver::got_proof(td::BufferSlice data) {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::written_proof);
   });
-  td::actor::send_closure(archive_, &ArchiveManager::write, handle_->unix_time(), handle_->is_key_block(),
-                          FileDb::RefId{fileref::Proof{block_id_}}, std::move(data), std::move(P));
+  td::actor::send_closure(archive_, &ArchiveManager::add_file, handle_, fileref::Proof{handle_->id()}, std::move(data),
+                          std::move(P));
 }
 
 void BlockArchiver::written_proof() {
@@ -95,12 +80,9 @@ void BlockArchiver::written_proof() {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::got_proof_link, R.move_as_ok());
   });
-  if (handle_->moved_to_storage()) {
-    td::actor::send_closure(archive_db_, &FileDb::load_file, FileDb::RefId{fileref::ProofLink{block_id_}},
-                            std::move(P));
-  } else {
-    td::actor::send_closure(file_db_, &FileDb::load_file, FileDb::RefId{fileref::ProofLink{block_id_}}, std::move(P));
-  }
+
+  td::actor::send_closure(archive_, &ArchiveManager::get_temp_file_short, fileref::ProofLink{handle_->id()},
+                          std::move(P));
 }
 
 void BlockArchiver::got_proof_link(td::BufferSlice data) {
@@ -108,8 +90,8 @@ void BlockArchiver::got_proof_link(td::BufferSlice data) {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::written_proof_link);
   });
-  td::actor::send_closure(archive_, &ArchiveManager::write, handle_->unix_time(), handle_->is_key_block(),
-                          FileDb::RefId{fileref::ProofLink{block_id_}}, std::move(data), std::move(P));
+  td::actor::send_closure(archive_, &ArchiveManager::add_file, handle_, fileref::ProofLink{handle_->id()},
+                          std::move(data), std::move(P));
 }
 
 void BlockArchiver::written_proof_link() {
@@ -121,11 +103,8 @@ void BlockArchiver::written_proof_link() {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::got_block_data, R.move_as_ok());
   });
-  if (handle_->moved_to_storage()) {
-    td::actor::send_closure(archive_db_, &FileDb::load_file, FileDb::RefId{fileref::Block{block_id_}}, std::move(P));
-  } else {
-    td::actor::send_closure(file_db_, &FileDb::load_file, FileDb::RefId{fileref::Block{block_id_}}, std::move(P));
-  }
+
+  td::actor::send_closure(archive_, &ArchiveManager::get_temp_file_short, fileref::Block{handle_->id()}, std::move(P));
 }
 
 void BlockArchiver::got_block_data(td::BufferSlice data) {
@@ -133,8 +112,8 @@ void BlockArchiver::got_block_data(td::BufferSlice data) {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::written_block_data);
   });
-  td::actor::send_closure(archive_, &ArchiveManager::write, handle_->unix_time(), handle_->is_key_block(),
-                          FileDb::RefId{fileref::Block{block_id_}}, std::move(data), std::move(P));
+  td::actor::send_closure(archive_, &ArchiveManager::add_file, handle_, fileref::Block{handle_->id()}, std::move(data),
+                          std::move(P));
 }
 
 void BlockArchiver::written_block_data() {
@@ -144,7 +123,7 @@ void BlockArchiver::written_block_data() {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::finish_query);
   });
-  td::actor::send_closure(root_db_, &RootDb::store_block_handle, handle_, std::move(P));
+  td::actor::send_closure(archive_, &ArchiveManager::update_handle, handle_, std::move(P));
 }
 
 void BlockArchiver::finish_query() {
@@ -156,7 +135,7 @@ void BlockArchiver::finish_query() {
 
 void BlockArchiver::abort_query(td::Status reason) {
   if (promise_) {
-    VLOG(VALIDATOR_WARNING) << "failed to archive block " << block_id_ << ": " << reason;
+    VLOG(VALIDATOR_WARNING) << "failed to archive block " << handle_->id() << ": " << reason;
     promise_.set_error(std::move(reason));
   }
   stop();

validator/db/archiver.hpp

@@ -22,7 +22,7 @@
 #include "td/actor/actor.h"
 #include "validator/interfaces/block-handle.h"
 #include "ton/ton-io.hpp"
-#include "archive-db.hpp"
+#include "archive-manager.hpp"
 
 namespace ton {
 
@@ -33,14 +33,12 @@ class FileDb;
 
 class BlockArchiver : public td::actor::Actor {
  public:
-  BlockArchiver(BlockIdExt block_id, td::actor::ActorId<RootDb> root_db, td::actor::ActorId<FileDb> file_db,
-                td::actor::ActorId<FileDb> archive_db, td::actor::ActorId<ArchiveManager> archive,
-                td::Promise<td::Unit> promise);
+  BlockArchiver(BlockHandle handle, td::actor::ActorId<ArchiveManager> archive_db, td::Promise<td::Unit> promise);
 
   void abort_query(td::Status error);
 
   void start_up() override;
-  void got_block_handle(BlockHandle handle);
+  void moved_handle();
   void got_proof(td::BufferSlice data);
   void written_proof();
   void got_proof_link(td::BufferSlice data);
@@ -50,14 +48,9 @@ class BlockArchiver : public td::actor::Actor {
   void finish_query();
 
  private:
-  BlockIdExt block_id_;
-  td::actor::ActorId<RootDb> root_db_;
-  td::actor::ActorId<FileDb> file_db_;
-  td::actor::ActorId<FileDb> archive_db_;
+  BlockHandle handle_;
   td::actor::ActorId<ArchiveManager> archive_;
   td::Promise<td::Unit> promise_;
-
-  BlockHandle handle_;
 };
 
 }  // namespace validator

validator/db/blockdb.cpp

@@ -1,301 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#include "blockdb.hpp"
-#include "rootdb.hpp"
-#include "validator/fabric.h"
-#include "ton/ton-tl.hpp"
-#include "td/utils/port/path.h"
-#include "files-async.hpp"
-#include "td/db/RocksDb.h"
-
-namespace ton {
-
-namespace validator {
-
-void BlockDb::store_block_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
-  if (!handle->id().is_valid()) {
-    promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid block id"));
-    return;
-  }
-  auto lru_key = get_block_lru_key(handle->id());
-  auto value_key = get_block_value_key(handle->id());
-  while (handle->need_flush()) {
-    auto version = handle->version();
-    auto R = get_block_value(value_key);
-    if (R.is_ok()) {
-      kv_->begin_transaction().ensure();
-      set_block_value(value_key, handle->serialize());
-      kv_->commit_transaction().ensure();
-    } else {
-      CHECK(get_block_lru(lru_key).is_error());
-      auto empty = get_block_lru_empty_key_hash();
-      auto ER = get_block_lru(empty);
-      ER.ensure();
-      auto E = ER.move_as_ok();
-
-      auto PR = get_block_lru(E.prev);
-      PR.ensure();
-      auto P = PR.move_as_ok();
-      CHECK(P.next == empty);
-
-      DbEntry D{handle->id(), E.prev, empty};
-
-      E.prev = lru_key;
-      P.next = lru_key;
-
-      if (P.is_empty()) {
-        E.next = lru_key;
-        P.prev = lru_key;
-      }
-
-      kv_->begin_transaction().ensure();
-      set_block_value(value_key, handle->serialize());
-      set_block_lru(empty, std::move(E));
-      set_block_lru(D.prev, std::move(P));
-      set_block_lru(lru_key, std::move(D));
-      kv_->commit_transaction().ensure();
-    }
-    handle->flushed_upto(version);
-  }
-  promise.set_value(td::Unit());
-}
-
-void BlockDb::get_block_handle(BlockIdExt id, td::Promise<BlockHandle> promise) {
-  if (!id.is_valid()) {
-    promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid block id"));
-    return;
-  }
-  CHECK(id.is_valid());
-  auto key_hash = get_block_value_key(id);
-  auto B = get_block_value(key_hash);
-  if (B.is_error()) {
-    promise.set_error(B.move_as_error());
-    return;
-  }
-  promise.set_result(create_block_handle(B.move_as_ok()));
-}
-
-void BlockDb::start_up() {
-  kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
-  alarm_timestamp() = td::Timestamp::in(0.1);
-
-  auto empty = get_block_lru_empty_key_hash();
-  if (get_block_lru(empty).is_error()) {
-    DbEntry e{BlockIdExt{}, empty, empty};
-    kv_->begin_transaction().ensure();
-    set_block_lru(empty, std::move(e));
-    kv_->commit_transaction().ensure();
-  }
-  last_gc_ = empty;
-}
-
-BlockDb::BlockDb(td::actor::ActorId<RootDb> root_db, std::string db_path) : root_db_(root_db), db_path_(db_path) {
-}
-
-BlockDb::KeyHash BlockDb::get_block_lru_key(BlockIdExt id) {
-  if (!id.is_valid()) {
-    return KeyHash::zero();
-  } else {
-    auto obj = create_tl_object<ton_api::db_blockdb_key_lru>(create_tl_block_id(id));
-    return get_tl_object_sha_bits256(obj);
-  }
-}
-
-BlockDb::KeyHash BlockDb::get_block_value_key(BlockIdExt id) {
-  CHECK(id.is_valid());
-  auto obj = create_tl_object<ton_api::db_blockdb_key_value>(create_tl_block_id(id));
-  return get_tl_object_sha_bits256(obj);
-}
-
-td::Result<BlockDb::DbEntry> BlockDb::get_block_lru(KeyHash key_hash) {
-  std::string value;
-  auto R = kv_->get(key_hash.as_slice(), value);
-  R.ensure();
-  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-    return td::Status::Error(ErrorCode::notready, "not in db");
-  }
-  auto v = fetch_tl_object<ton_api::db_blockdb_lru>(td::BufferSlice{value}, true);
-  v.ensure();
-  return DbEntry{v.move_as_ok()};
-}
-
-td::Result<td::BufferSlice> BlockDb::get_block_value(KeyHash key_hash) {
-  std::string value;
-  auto R = kv_->get(key_hash.as_slice(), value);
-  R.ensure();
-  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-    return td::Status::Error(ErrorCode::notready, "not in db");
-  }
-  return td::BufferSlice{value};
-}
-
-void BlockDb::set_block_lru(KeyHash key_hash, DbEntry e) {
-  kv_->set(key_hash.as_slice(), serialize_tl_object(e.release(), true)).ensure();
-}
-
-void BlockDb::set_block_value(KeyHash key_hash, td::BufferSlice value) {
-  kv_->set(key_hash.as_slice(), std::move(value)).ensure();
-}
-
-void BlockDb::alarm() {
-  auto R = get_block_lru(last_gc_);
-  R.ensure();
-
-  auto N = R.move_as_ok();
-  if (N.is_empty()) {
-    last_gc_ = N.next;
-    alarm_timestamp() = td::Timestamp::in(0.01);
-    return;
-  }
-
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<bool> R) {
-    if (R.is_error()) {
-      td::actor::send_closure(SelfId, &BlockDb::skip_gc);
-    } else {
-      auto value = R.move_as_ok();
-      if (!value) {
-        td::actor::send_closure(SelfId, &BlockDb::skip_gc);
-      } else {
-        td::actor::send_closure(SelfId, &BlockDb::gc);
-      }
-    }
-  });
-  td::actor::send_closure(root_db_, &RootDb::allow_block_gc, N.block_id, std::move(P));
-}
-
-void BlockDb::gc() {
-  auto FR = get_block_lru(last_gc_);
-  FR.ensure();
-  auto F = FR.move_as_ok();
-
-  auto PR = get_block_lru(F.prev);
-  PR.ensure();
-  auto P = PR.move_as_ok();
-  auto NR = get_block_lru(F.next);
-  NR.ensure();
-  auto N = NR.move_as_ok();
-
-  P.next = F.next;
-  N.prev = F.prev;
-  if (P.is_empty() && N.is_empty()) {
-    P.prev = P.next;
-    N.next = N.prev;
-  }
-
-  auto value_key = get_block_value_key(F.block_id);
-
-  kv_->begin_transaction().ensure();
-  kv_->erase(last_gc_.as_slice()).ensure();
-  kv_->erase(value_key.as_slice()).ensure();
-  set_block_lru(F.prev, std::move(P));
-  set_block_lru(F.next, std::move(N));
-  kv_->commit_transaction().ensure();
-  alarm_timestamp() = td::Timestamp::now();
-
-  DCHECK(get_block_lru(last_gc_).is_error());
-  last_gc_ = F.next;
-}
-
-void BlockDb::skip_gc() {
-  auto R = get_block_lru(last_gc_);
-  R.ensure();
-
-  auto N = R.move_as_ok();
-  last_gc_ = N.next;
-  alarm_timestamp() = td::Timestamp::in(0.01);
-}
-
-BlockDb::DbEntry::DbEntry(tl_object_ptr<ton_api::db_blockdb_lru> entry)
-    : block_id(create_block_id(entry->id_)), prev(entry->prev_), next(entry->next_) {
-}
-
-tl_object_ptr<ton_api::db_blockdb_lru> BlockDb::DbEntry::release() {
-  return create_tl_object<ton_api::db_blockdb_lru>(create_tl_block_id(block_id), prev, next);
-}
-
-void BlockDb::truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) {
-  std::map<ShardIdFull, BlockSeqno> max_seqno;
-  max_seqno.emplace(ShardIdFull{masterchainId}, state->get_seqno() + 1);
-
-  auto shards = state->get_shards();
-  auto it = KeyHash::zero();
-  kv_->begin_transaction().ensure();
-  while (true) {
-    auto R = get_block_lru(it);
-    R.ensure();
-    auto v = R.move_as_ok();
-    it = v.next;
-    R = get_block_lru(it);
-    R.ensure();
-    v = R.move_as_ok();
-    if (v.is_empty()) {
-      break;
-    }
-
-    auto s = v.block_id.shard_full();
-    if (!max_seqno.count(s)) {
-      bool found = false;
-      for (auto &shard : shards) {
-        if (shard_intersects(shard->shard(), s)) {
-          found = true;
-          max_seqno.emplace(s, shard->top_block_id().seqno() + 1);
-          break;
-        }
-      }
-      if (!found) {
-        max_seqno.emplace(s, 0);
-      }
-    }
-
-    bool to_delete = v.block_id.seqno() >= max_seqno[s];
-    if (to_delete) {
-      auto key_hash = get_block_value_key(v.block_id);
-      auto B = get_block_value(key_hash);
-      B.ensure();
-      auto handleR = create_block_handle(B.move_as_ok());
-      handleR.ensure();
-      auto handle = handleR.move_as_ok();
-
-      handle->unsafe_clear_applied();
-      handle->unsafe_clear_next();
-
-      if (handle->need_flush()) {
-        set_block_value(key_hash, handle->serialize());
-      }
-    } else if (v.block_id.seqno() + 1 == max_seqno[s]) {
-      auto key_hash = get_block_value_key(v.block_id);
-      auto B = get_block_value(key_hash);
-      B.ensure();
-      auto handleR = create_block_handle(B.move_as_ok());
-      handleR.ensure();
-      auto handle = handleR.move_as_ok();
-
-      handle->unsafe_clear_next();
-
-      if (handle->need_flush()) {
-        set_block_value(key_hash, handle->serialize());
-      }
-    }
-  }
-  kv_->commit_transaction().ensure();
-}
-
-}  // namespace validator
-
-}  // namespace ton

validator/db/blockdb.hpp

@@ -1,88 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#pragma once
-
-#include "td/actor/actor.h"
-#include "ton/ton-types.h"
-#include "validator/interfaces/block-handle.h"
-#include "validator/interfaces/db.h"
-#include "td/db/KeyValueAsync.h"
-
-namespace ton {
-
-namespace validator {
-
-class RootDb;
-
-class BlockDb : public td::actor::Actor {
- public:
-  void store_block_handle(BlockHandle handle, td::Promise<td::Unit> promise);
-  void get_block_handle(BlockIdExt id, td::Promise<BlockHandle> promise);
-
-  void start_up() override;
-  void alarm() override;
-
-  void gc();
-  void skip_gc();
-
-  void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise);
-
-  BlockDb(td::actor::ActorId<RootDb> root_db, std::string db_path);
-
- private:
-  using KeyHash = td::Bits256;
-  struct DbEntry {
-    BlockIdExt block_id;
-    KeyHash prev;
-    KeyHash next;
-
-    DbEntry(tl_object_ptr<ton_api::db_blockdb_lru> entry);
-    DbEntry() {
-    }
-    DbEntry(BlockIdExt block_id, KeyHash prev, KeyHash next) : block_id(block_id), prev(prev), next(next) {
-    }
-    tl_object_ptr<ton_api::db_blockdb_lru> release();
-    bool is_empty() const {
-      return !block_id.is_valid();
-    }
-  };
-  static KeyHash get_block_lru_key(BlockIdExt block_id);
-  static KeyHash get_block_value_key(BlockIdExt block_id);
-  static KeyHash get_block_lru_empty_key_hash() {
-    return KeyHash::zero();
-  }
-
-  td::Result<DbEntry> get_block_lru(KeyHash key);
-  td::Result<td::BufferSlice> get_block_value(KeyHash key);
-
-  void set_block_lru(KeyHash key_hash, DbEntry e);
-  void set_block_value(KeyHash key_hash, td::BufferSlice data);
-
-  std::shared_ptr<td::KeyValue> kv_;
-
-  td::actor::ActorId<RootDb> root_db_;
-
-  std::string db_path_;
-
-  KeyHash last_gc_ = KeyHash::zero();
-};
-
-}  // namespace validator
-
-}  // namespace ton

validator/db/filedb.cpp

@@ -1,400 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#include "filedb.hpp"
-#include "rootdb.hpp"
-#include "td/utils/port/path.h"
-#include "files-async.hpp"
-#include "adnl/utils.hpp"
-#include "tl-utils/tl-utils.hpp"
-#include "td/utils/overloaded.h"
-
-#include "td/db/RocksDb.h"
-
-namespace ton {
-
-namespace validator {
-
-std::string FileDb::get_file_name(const RefId& ref_id, bool create_dirs) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-
-  auto s = ref_id_hash.to_hex();
-  std::string path = root_path_ + "/files/";
-  for (td::uint32 i = 0; i < depth_; i++) {
-    path = path + s[2 * i] + s[2 * i + 1] + "/";
-    if (create_dirs) {
-      td::mkdir(path).ensure();
-    }
-  }
-  return path + s;
-}
-
-void FileDb::store_file(RefId ref_id, td::BufferSlice data, td::Promise<FileHash> promise) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-  auto R = get_block(ref_id_hash);
-  if (R.is_ok()) {
-    auto val = R.move_as_ok();
-    promise.set_result(val.file_hash);
-    return;
-  }
-
-  auto file_hash = sha256_bits256(data.as_slice());
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), file_hash, ref_id = std::move(ref_id),
-                                       promise = std::move(promise)](td::Result<std::string> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      td::actor::send_closure(SelfId, &FileDb::store_file_continue, std::move(ref_id), file_hash, R.move_as_ok(),
-                              std::move(promise));
-    }
-  });
-  td::actor::create_actor<db::WriteFile>("writefile", root_path_ + "/tmp/", "", std::move(data), std::move(P))
-      .release();
-}
-
-void FileDb::store_file_continue(RefId ref_id, FileHash file_hash, std::string res_path,
-                                 td::Promise<FileHash> promise) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-  auto R = get_block(ref_id_hash);
-  if (R.is_ok()) {
-    td::unlink(res_path).ignore();
-    auto val = R.move_as_ok();
-    promise.set_result(val.file_hash);
-    return;
-  }
-
-  auto path = get_file_name(ref_id, true);
-  td::rename(res_path, path).ensure();
-
-  auto empty = get_empty_ref_id_hash();
-  auto ER = get_block(empty);
-  ER.ensure();
-  auto E = ER.move_as_ok();
-
-  auto PR = get_block(E.prev);
-  PR.ensure();
-  auto P = PR.move_as_ok();
-  CHECK(P.next == empty);
-
-  DbEntry D;
-  D.key = std::move(ref_id);
-  D.prev = E.prev;
-  D.next = empty;
-  D.file_hash = file_hash;
-
-  E.prev = ref_id_hash;
-  P.next = ref_id_hash;
-
-  if (P.is_empty()) {
-    E.next = ref_id_hash;
-    P.prev = ref_id_hash;
-  }
-
-  kv_->begin_transaction().ensure();
-  set_block(empty, std::move(E));
-  set_block(D.prev, std::move(P));
-  set_block(ref_id_hash, std::move(D));
-  kv_->commit_transaction().ensure();
-
-  promise.set_value(std::move(file_hash));
-}
-
-void FileDb::load_file(RefId ref_id, td::Promise<td::BufferSlice> promise) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-  auto R = get_block(ref_id_hash);
-  if (R.is_error()) {
-    promise.set_error(R.move_as_error());
-    return;
-  }
-
-  auto v = R.move_as_ok();
-
-  auto P = td::PromiseCreator::lambda(
-      [promise = std::move(promise), file_hash = v.file_hash](td::Result<td::BufferSlice> R) mutable {
-        if (R.is_error()) {
-          promise.set_error(R.move_as_error());
-        } else {
-          auto data = R.move_as_ok();
-          if (file_hash != sha256_bits256(data.as_slice())) {
-            promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "db error: bad file hash"));
-          } else {
-            promise.set_value(std::move(data));
-          }
-        }
-      });
-
-  td::actor::create_actor<db::ReadFile>("readfile", get_file_name(ref_id, false), 0, -1, 0, std::move(P)).release();
-}
-
-void FileDb::load_file_slice(RefId ref_id, td::int64 offset, td::int64 max_size, td::Promise<td::BufferSlice> promise) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-  auto R = get_block(ref_id_hash);
-  if (R.is_error()) {
-    promise.set_error(R.move_as_error());
-    return;
-  }
-
-  auto v = R.move_as_ok();
-
-  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      promise.set_value(R.move_as_ok());
-    }
-  });
-
-  td::actor::create_actor<db::ReadFile>("readfile", get_file_name(ref_id, false), offset, max_size, 0, std::move(P))
-      .release();
-}
-
-void FileDb::check_file(RefId ref_id, td::Promise<bool> promise) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-  auto R = get_block(ref_id_hash);
-  if (R.is_error()) {
-    promise.set_result(false);
-    return;
-  }
-  promise.set_result(true);
-}
-
-td::Slice FileDb::get_key(const RefIdHash& ref) {
-  return ref.as_slice();
-}
-
-td::Result<FileDb::DbEntry> FileDb::get_block(const RefIdHash& ref) {
-  std::string value;
-  auto R = kv_->get(get_key(ref), value);
-  R.ensure();
-  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-    return td::Status::Error(ErrorCode::notready, "not in db");
-  }
-  auto val = fetch_tl_object<ton_api::db_filedb_value>(td::BufferSlice{value}, true);
-  val.ensure();
-  return DbEntry{val.move_as_ok()};
-}
-
-void FileDb::set_block(const RefIdHash& ref, DbEntry entry) {
-  DCHECK(ref == get_ref_id_hash(entry.key));
-  kv_->set(get_key(ref), entry.release()).ensure();
-}
-
-FileDb::FileDb(td::actor::ActorId<RootDb> root_db, std::string root_path, td::uint32 depth, bool is_archive)
-    : root_db_(root_db), root_path_(root_path), depth_(depth), is_archive_(is_archive) {
-}
-
-void FileDb::start_up() {
-  td::mkdir(root_path_).ensure();
-  db_path_ = root_path_ + "/db/";
-
-  kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
-  td::mkdir(root_path_ + "/files/").ensure();
-  td::mkdir(root_path_ + "/tmp/").ensure();
-
-  last_gc_ = get_empty_ref_id_hash();
-  alarm_timestamp() = td::Timestamp::in(0.01);
-
-  auto R = get_block(last_gc_);
-  if (R.is_error()) {
-    DbEntry e{get_empty_ref_id(), last_gc_, last_gc_, FileHash::zero()};
-    kv_->begin_transaction().ensure();
-    set_block(last_gc_, std::move(e));
-    kv_->commit_transaction().ensure();
-  }
-}
-
-void FileDb::alarm() {
-  auto R = get_block(last_gc_);
-  R.ensure();
-  auto N = R.move_as_ok();
-  if (N.is_empty()) {
-    last_gc_ = N.next;
-    alarm_timestamp() = td::Timestamp::in(0.01);
-    return;
-  }
-
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<bool> R) mutable {
-    if (R.is_error()) {
-      td::actor::send_closure(SelfId, &FileDb::skip_gc);
-    } else {
-      auto value = R.move_as_ok();
-      if (!value) {
-        td::actor::send_closure(SelfId, &FileDb::skip_gc);
-      } else {
-        td::actor::send_closure(SelfId, &FileDb::gc);
-      }
-    }
-  });
-  td::actor::send_closure(root_db_, &RootDb::allow_gc, std::move(N.key), is_archive_, std::move(P));
-}
-
-void FileDb::gc() {
-  auto FR = get_block(last_gc_);
-  FR.ensure();
-  auto F = FR.move_as_ok();
-
-  auto PR = get_block(F.prev);
-  PR.ensure();
-  auto P = PR.move_as_ok();
-  auto NR = get_block(F.next);
-  NR.ensure();
-  auto N = NR.move_as_ok();
-
-  P.next = F.next;
-  N.prev = F.prev;
-  if (P.is_empty() && N.is_empty()) {
-    P.prev = P.next;
-    N.next = N.prev;
-  }
-
-  auto name = get_file_name(F.key, false);
-  auto S = td::unlink(name);
-  if (S.is_error()) {
-    LOG(WARNING) << "failed to delete " << name;
-  }
-
-  kv_->begin_transaction().ensure();
-  kv_->erase(last_gc_.as_slice()).ensure();
-  set_block(F.prev, std::move(P));
-  set_block(F.next, std::move(N));
-  kv_->commit_transaction().ensure();
-  alarm_timestamp() = td::Timestamp::now();
-
-  DCHECK(get_block(last_gc_).is_error());
-  last_gc_ = F.next;
-}
-
-void FileDb::skip_gc() {
-  auto FR = get_block(last_gc_);
-  FR.ensure();
-  auto F = FR.move_as_ok();
-  last_gc_ = F.next;
-  alarm_timestamp() = td::Timestamp::in(0.01);
-}
-
-td::BufferSlice FileDb::DbEntry::release() {
-  return create_serialize_tl_object<ton_api::db_filedb_value>(get_ref_id_tl(key), prev, next, file_hash);
-}
-
-bool FileDb::DbEntry::is_empty() const {
-  return (key.get_offset() == key.offset<fileref::Empty>());
-}
-
-FileDb::RefIdHash FileDb::get_ref_id_hash(const RefId& ref) {
-  FileHash x;
-  ref.visit([&](const auto& obj) { x = obj.hash(); });
-  return x;
-}
-
-tl_object_ptr<ton_api::db_filedb_Key> FileDb::get_ref_id_tl(const RefId& ref) {
-  tl_object_ptr<ton_api::db_filedb_Key> x;
-  ref.visit([&](const auto& obj) { x = obj.tl(); });
-  return x;
-}
-
-FileDb::RefId FileDb::get_ref_from_tl(const ton_api::db_filedb_Key& from) {
-  RefId ref_id{fileref::Empty{}};
-  ton_api::downcast_call(
-      const_cast<ton_api::db_filedb_Key&>(from),
-      td::overloaded(
-          [&](const ton_api::db_filedb_key_empty& key) { ref_id = fileref::Empty{}; },
-          [&](const ton_api::db_filedb_key_blockFile& key) { ref_id = fileref::Block{create_block_id(key.block_id_)}; },
-          [&](const ton_api::db_filedb_key_zeroStateFile& key) {
-            ref_id = fileref::ZeroState{create_block_id(key.block_id_)};
-          },
-          [&](const ton_api::db_filedb_key_persistentStateFile& key) {
-            ref_id =
-                fileref::PersistentState{create_block_id(key.block_id_), create_block_id(key.masterchain_block_id_)};
-          },
-          [&](const ton_api::db_filedb_key_proof& key) { ref_id = fileref::Proof{create_block_id(key.block_id_)}; },
-          [&](const ton_api::db_filedb_key_proofLink& key) {
-            ref_id = fileref::ProofLink{create_block_id(key.block_id_)};
-          },
-          [&](const ton_api::db_filedb_key_signatures& key) {
-            ref_id = fileref::Signatures{create_block_id(key.block_id_)};
-          },
-          [&](const ton_api::db_filedb_key_candidate& key) {
-            ref_id = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_),
-                                        key.id_->collated_data_file_hash_};
-          },
-          [&](const ton_api::db_filedb_key_blockInfo& key) {
-            ref_id = fileref::BlockInfo{create_block_id(key.block_id_)};
-          }));
-  return ref_id;
-}
-
-FileDb::RefId FileDb::get_empty_ref_id() {
-  return fileref::Empty();
-}
-
-FileDb::RefIdHash FileDb::get_empty_ref_id_hash() {
-  if (empty_.is_zero()) {
-    empty_ = get_ref_id_hash(get_empty_ref_id());
-  }
-  return empty_;
-}
-
-FileDb::DbEntry::DbEntry(tl_object_ptr<ton_api::db_filedb_value> entry)
-    : key(FileDb::get_ref_from_tl(*entry->key_.get()))
-    , prev(entry->prev_)
-    , next(entry->next_)
-    , file_hash(entry->file_hash_) {
-}
-
-void FileDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
-  std::vector<std::pair<std::string, std::string>> rocksdb_stats;
-  auto stats = kv_->stats();
-  if (stats.size() == 0) {
-    promise.set_value(std::move(rocksdb_stats));
-    return;
-  }
-  size_t pos = 0;
-  while (pos < stats.size()) {
-    while (pos < stats.size() &&
-           (stats[pos] == ' ' || stats[pos] == '\n' || stats[pos] == '\r' || stats[pos] == '\t')) {
-      pos++;
-    }
-    auto p = pos;
-    if (pos == stats.size()) {
-      break;
-    }
-    while (stats[pos] != '\n' && stats[pos] != '\r' && stats[pos] != ' ' && stats[pos] != '\t' && pos < stats.size()) {
-      pos++;
-    }
-    auto name = stats.substr(p, pos - p);
-    if (stats[pos] == '\n' || pos == stats.size()) {
-      rocksdb_stats.emplace_back(name, "");
-      continue;
-    }
-    while (pos < stats.size() &&
-           (stats[pos] == ' ' || stats[pos] == '\n' || stats[pos] == '\r' || stats[pos] == '\t')) {
-      pos++;
-    }
-    p = pos;
-    while (stats[pos] != '\n' && stats[pos] != '\r' && pos < stats.size()) {
-      pos++;
-    }
-    auto value = stats.substr(p, pos - p);
-    rocksdb_stats.emplace_back(name, value);
-  }
-  promise.set_value(std::move(rocksdb_stats));
-}
-
-}  // namespace validator
-
-}  // namespace ton

validator/db/filedb.hpp

@@ -1,218 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#pragma once
-
-#include "ton/ton-types.h"
-#include "td/actor/actor.h"
-#include "validator/interfaces/shard.h"
-#include "td/db/KeyValue.h"
-#include "ton/ton-tl.hpp"
-
-namespace ton {
-
-namespace validator {
-
-namespace fileref {
-
-class Empty {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_empty>();
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_empty>();
-  }
-};
-
-class Block {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_blockFile>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_blockFile>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-
-class ZeroState {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_zeroStateFile>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_zeroStateFile>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-
-class PersistentState {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_persistentStateFile>(create_tl_block_id(block_id),
-                                                                        create_tl_block_id(masterchain_block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_persistentStateFile>(create_tl_block_id(block_id),
-                                                                             create_tl_block_id(masterchain_block_id));
-  }
-
-  BlockIdExt block_id;
-  BlockIdExt masterchain_block_id;
-};
-
-class Proof {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_proof>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_proof>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-
-class ProofLink {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_proofLink>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_proofLink>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-
-class Signatures {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_signatures>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_signatures>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-
-class Candidate {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_candidate>(
-        create_tl_object<ton_api::db_candidate_id>(source.tl(), create_tl_block_id(block_id), collated_data_file_hash));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_candidate>(
-        create_tl_object<ton_api::db_candidate_id>(source.tl(), create_tl_block_id(block_id), collated_data_file_hash));
-  }
-
-  PublicKey source;
-  BlockIdExt block_id;
-  FileHash collated_data_file_hash;
-};
-
-class BlockInfo {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_blockInfo>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_blockInfo>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-};  // namespace fileref
-
-class RootDb;
-
-class FileDb : public td::actor::Actor {
- public:
-  using RefId =
-      td::Variant<fileref::Empty, fileref::Block, fileref::ZeroState, fileref::PersistentState, fileref::Proof,
-                  fileref::Proof, fileref::ProofLink, fileref::Signatures, fileref::Candidate, fileref::BlockInfo>;
-  using RefIdHash = td::Bits256;
-
-  void store_file(RefId ref_id, td::BufferSlice data, td::Promise<FileHash> promise);
-  void store_file_continue(RefId ref_id, FileHash file_hash, std::string path, td::Promise<FileHash> promise);
-  void load_file(RefId ref_id, td::Promise<td::BufferSlice> promise);
-  void load_file_slice(RefId ref_id, td::int64 offset, td::int64 max_size, td::Promise<td::BufferSlice> promise);
-  void check_file(RefId ref_id, td::Promise<bool> promise);
-
-  void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise);
-
-  void start_up() override;
-  void alarm() override;
-
-  void gc();
-  void skip_gc();
-
-  FileDb(td::actor::ActorId<RootDb> root_db, std::string root_path, td::uint32 depth, bool is_archive);
-
- private:
-  struct DbEntry {
-    RefId key;
-    RefIdHash prev;
-    RefIdHash next;
-    FileHash file_hash;
-
-    DbEntry(tl_object_ptr<ton_api::db_filedb_value> entry);
-    DbEntry() {
-    }
-    DbEntry(RefId key, RefIdHash prev, RefIdHash next, FileHash file_hash)
-        : key(std::move(key)), prev(prev), next(next), file_hash(file_hash) {
-    }
-    td::BufferSlice release();
-    bool is_empty() const;
-  };
-
-  static RefIdHash get_ref_id_hash(const RefId& ref);
-  static tl_object_ptr<ton_api::db_filedb_Key> get_ref_id_tl(const RefId& ref);
-  static RefId get_ref_from_tl(const ton_api::db_filedb_Key& from);
-  static RefId get_empty_ref_id();
-  RefIdHash get_empty_ref_id_hash();
-
-  std::string get_file_name(const RefId& ref, bool create_dirs);
-  td::Slice get_key(const RefIdHash& ref);
-
-  td::Result<DbEntry> get_block(const RefIdHash& ref_id);
-  void set_block(const RefIdHash& ref_id_hash, DbEntry entry);
-
-  td::actor::ActorId<RootDb> root_db_;
-
-  std::string root_path_;
-  std::string db_path_;
-  td::uint32 depth_;
-
-  bool is_archive_;
-
-  std::shared_ptr<td::KeyValue> kv_;
-
-  RefIdHash last_gc_;
-  RefIdHash empty_ = RefIdHash::zero();
-};
-
-}  // namespace validator
-
-}  // namespace ton
-

validator/db/fileref.cpp

@@ -0,0 +1,471 @@
+#include "fileref.hpp"
+#include "auto/tl/ton_api.hpp"
+#include "td/utils/overloaded.h"
+#include "td/utils/misc.h"
+
+namespace ton {
+
+namespace validator {
+
+namespace {
+
+td::Result<BlockId> get_block_id(std::stringstream& ss) {
+  std::string token;
+  BlockId block_id;
+  std::getline(ss, token, '_');
+  TRY_RESULT(w, td::to_integer_safe<WorkchainId>(token));
+  std::getline(ss, token, '_');
+  TRY_RESULT(shard, td::hex_to_integer_safe<ShardId>(token));
+  std::getline(ss, token, '_');
+  TRY_RESULT(s, td::to_integer_safe<BlockSeqno>(token));
+  return BlockId{w, shard, s};
+}
+
+td::Result<FileHash> get_token_hash(std::stringstream& ss) {
+  std::string token;
+  std::getline(ss, token, '_');
+  if (token.size() != 64) {
+    return td::Status::Error(ErrorCode::protoviolation, "hash must have exactly 64 hexdigits");
+  }
+
+  TRY_RESULT(v, td::hex_decode(token));
+
+  FileHash r;
+  r.as_slice().copy_from(v);
+  return r;
+}
+
+}  // namespace
+
+namespace fileref {
+
+std::string Empty::filename() const {
+  return "empty";
+}
+
+Empty Empty::shortref() const {
+  return *this;
+}
+
+std::string Empty::filename_short() const {
+  return "empty";
+}
+
+BlockShort Block::shortref() const {
+  return BlockShort{block_id.id, hash()};
+}
+
+std::string Block::filename() const {
+  return PSTRING() << "block_" << block_id.to_str();
+}
+
+std::string Block::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "block_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string BlockShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "block_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
+}
+
+ZeroStateShort ZeroState::shortref() const {
+  return ZeroStateShort{block_id.id.workchain, hash()};
+}
+
+std::string ZeroState::filename() const {
+  return PSTRING() << "zerostate_" << block_id.to_str();
+}
+
+std::string ZeroState::filename_short() const {
+  return PSTRING() << "zerostate_" << block_id.id.workchain << "_" << hash().to_hex();
+}
+
+std::string ZeroStateShort::filename_short() const {
+  return PSTRING() << "zerostate_" << workchain << "_" << hash().to_hex();
+}
+
+PersistentStateShort PersistentState::shortref() const {
+  return PersistentStateShort{block_id.shard_full(), masterchain_block_id.seqno(), hash()};
+}
+
+std::string PersistentState::filename() const {
+  return PSTRING() << "state_" << masterchain_block_id.to_str() << "_" << block_id.to_str();
+}
+
+std::string PersistentState::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "state_" << masterchain_block_id.seqno() << "_" << block_id.id.workchain << "_" << s << "_"
+                   << hash().to_hex();
+}
+
+std::string PersistentStateShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(shard_id.shard));
+  return PSTRING() << "state_" << masterchain_seqno << "_" << shard_id.workchain << "_" << s << "_" << hash().to_hex();
+}
+
+ProofShort Proof::shortref() const {
+  return ProofShort{block_id.id, hash()};
+}
+
+std::string Proof::filename() const {
+  return PSTRING() << "proof_" << block_id.to_str();
+}
+
+std::string Proof::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "proof_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string ProofShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "proof_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
+}
+
+ProofLinkShort ProofLink::shortref() const {
+  return ProofLinkShort{block_id.id, hash()};
+}
+
+std::string ProofLink::filename() const {
+  return PSTRING() << "prooflink_" << block_id.to_str();
+}
+
+std::string ProofLink::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "prooflink_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string ProofLinkShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "prooflink_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
+}
+
+SignaturesShort Signatures::shortref() const {
+  return SignaturesShort{block_id.id, hash()};
+}
+
+std::string Signatures::filename() const {
+  return PSTRING() << "signatures_" << block_id.to_str();
+}
+
+std::string Signatures::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "signatures_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string SignaturesShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "signatures_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_"
+                   << hash().to_hex();
+}
+
+CandidateShort Candidate::shortref() const {
+  return CandidateShort{block_id.id, hash()};
+}
+
+std::string Candidate::filename() const {
+  return PSTRING() << "candidate_" << block_id.to_str() << "_" << collated_data_file_hash.to_hex() << "_"
+                   << td::base64url_encode(source.export_as_slice());
+}
+
+std::string Candidate::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "candidate_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string CandidateShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "candidate_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
+}
+
+BlockInfoShort BlockInfo::shortref() const {
+  return BlockInfoShort{block_id.id, hash()};
+}
+
+std::string BlockInfo::filename() const {
+  return PSTRING() << "info_" << block_id.to_str();
+}
+
+std::string BlockInfo::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "info_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string BlockInfoShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "info_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
+}
+
+}  // namespace fileref
+
+FileReference::FileReference(tl_object_ptr<ton_api::db_filedb_Key> key) {
+  ton_api::downcast_call(
+      *key.get(),
+      td::overloaded(
+          [&](const ton_api::db_filedb_key_empty& key) { ref_ = fileref::Empty{}; },
+          [&](const ton_api::db_filedb_key_blockFile& key) { ref_ = fileref::Block{create_block_id(key.block_id_)}; },
+          [&](const ton_api::db_filedb_key_zeroStateFile& key) {
+            ref_ = fileref::ZeroState{create_block_id(key.block_id_)};
+          },
+          [&](const ton_api::db_filedb_key_persistentStateFile& key) {
+            ref_ = fileref::PersistentState{create_block_id(key.block_id_), create_block_id(key.masterchain_block_id_)};
+          },
+          [&](const ton_api::db_filedb_key_proof& key) { ref_ = fileref::Proof{create_block_id(key.block_id_)}; },
+          [&](const ton_api::db_filedb_key_proofLink& key) {
+            ref_ = fileref::ProofLink{create_block_id(key.block_id_)};
+          },
+          [&](const ton_api::db_filedb_key_signatures& key) {
+            ref_ = fileref::Signatures{create_block_id(key.block_id_)};
+          },
+          [&](const ton_api::db_filedb_key_candidate& key) {
+            ref_ = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_),
+                                      key.id_->collated_data_file_hash_};
+          },
+          [&](const ton_api::db_filedb_key_blockInfo& key) {
+            ref_ = fileref::BlockInfo{create_block_id(key.block_id_)};
+          }));
+}
+
+FileReferenceShort FileReference::shortref() const {
+  FileReferenceShort h;
+  ref_.visit([&](const auto& obj) { h = obj.shortref(); });
+  return h;
+}
+
+td::Bits256 FileReference::hash() const {
+  FileHash h;
+  ref_.visit([&](const auto& obj) { h = obj.hash(); });
+  return h;
+}
+
+td::Bits256 FileReferenceShort::hash() const {
+  FileHash h;
+  ref_.visit([&](const auto& obj) { h = obj.hash(); });
+  return h;
+}
+
+ShardIdFull FileReference::shard() const {
+  ShardIdFull h;
+  ref_.visit([&](const auto& obj) { h = obj.shard(); });
+  return h;
+}
+
+ShardIdFull FileReferenceShort::shard() const {
+  ShardIdFull h;
+  ref_.visit([&](const auto& obj) { h = obj.shard(); });
+  return h;
+}
+
+std::string FileReference::filename() const {
+  std::string h;
+  ref_.visit([&](const auto& obj) { h = obj.filename(); });
+  return h;
+}
+
+std::string FileReference::filename_short() const {
+  std::string h;
+  ref_.visit([&](const auto& obj) { h = obj.filename_short(); });
+  return h;
+}
+
+std::string FileReferenceShort::filename_short() const {
+  std::string h;
+  ref_.visit([&](const auto& obj) { h = obj.filename_short(); });
+  return h;
+}
+
+td::Result<FileReference> FileReference::create(std::string filename) {
+  std::stringstream ss{filename};
+
+  std::string token;
+  std::getline(ss, token, '_');
+
+  if (token == "empty") {
+    if (ss.eof()) {
+      return fileref::Empty{};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "block") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::Block{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "zerostate") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::ZeroState{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "state") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(masterchain_block_id, BlockIdExt::from_str(token));
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::PersistentState{block_id, masterchain_block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "proof") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::Proof{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "prooflink") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::ProofLink{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "signatures") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::Signatures{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "candidate") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    TRY_RESULT(col_hash, get_token_hash(ss));
+
+    std::string rem = ss.str();
+
+    TRY_RESULT(source_s, td::base64url_decode(rem));
+    TRY_RESULT(source, PublicKey::import(source_s));
+    return fileref::Candidate{source, block_id, col_hash};
+  } else if (token == "info") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::BlockInfo{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else {
+    return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "unknown prefix '" << token << "'");
+  }
+}
+
+td::Result<FileReferenceShort> FileReferenceShort::create(std::string filename) {
+  std::stringstream ss{filename};
+
+  std::string token;
+  std::getline(ss, token, '_');
+
+  if (token == "empty") {
+    if (ss.eof()) {
+      return fileref::Empty{};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "block") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::BlockShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "zerostate") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(workchain, td::to_integer_safe<WorkchainId>(token));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::ZeroStateShort{workchain, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "state") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(masterchain_seqno, td::to_integer_safe<BlockSeqno>(token));
+    std::getline(ss, token, '_');
+    TRY_RESULT(workchain, td::to_integer_safe<WorkchainId>(token));
+    std::getline(ss, token, '_');
+    TRY_RESULT(shard, td::hex_to_integer_safe<ShardId>(token));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::PersistentStateShort{ShardIdFull{workchain, shard}, masterchain_seqno, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "proof") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::ProofShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "prooflink") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::ProofLinkShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "signatures") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::SignaturesShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "candidate") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::CandidateShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "info") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::BlockInfoShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else {
+    return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "unknown prefix '" << token << "'");
+  }
+}
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/fileref.hpp

@@ -0,0 +1,372 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "ton/ton-types.h"
+#include "td/actor/actor.h"
+#include "validator/interfaces/shard.h"
+#include "td/db/KeyValue.h"
+#include "ton/ton-tl.hpp"
+
+namespace ton {
+
+namespace validator {
+
+namespace fileref {
+
+class Empty {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_empty>();
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_empty>();
+  }
+  ShardIdFull shard() const {
+    return ShardIdFull{masterchainId};
+  }
+  std::string filename() const;
+  std::string filename_short() const;
+  Empty shortref() const;
+};
+
+class BlockShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class Block {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_blockFile>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_blockFile>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  BlockShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+class ZeroStateShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return ShardIdFull{workchain, shardIdAll};
+  }
+  std::string filename_short() const;
+
+  WorkchainId workchain;
+  FileHash hashv;
+};
+
+class ZeroState {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_zeroStateFile>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_zeroStateFile>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  ZeroStateShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+class PersistentStateShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return shard_id;
+  }
+  std::string filename_short() const;
+
+  ShardIdFull shard_id;
+  BlockSeqno masterchain_seqno;
+  FileHash hashv;
+};
+
+class PersistentState {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_persistentStateFile>(create_tl_block_id(block_id),
+                                                                        create_tl_block_id(masterchain_block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_persistentStateFile>(create_tl_block_id(block_id),
+                                                                             create_tl_block_id(masterchain_block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  PersistentStateShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+  BlockIdExt masterchain_block_id;
+};
+
+class ProofShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class Proof {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_proof>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_proof>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  ProofShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+class ProofLinkShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class ProofLink {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_proofLink>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_proofLink>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  ProofLinkShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+class SignaturesShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class Signatures {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_signatures>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_signatures>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  SignaturesShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+class CandidateShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class Candidate {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_candidate>(
+        create_tl_object<ton_api::db_candidate_id>(source.tl(), create_tl_block_id(block_id), collated_data_file_hash));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_candidate>(
+        create_tl_object<ton_api::db_candidate_id>(source.tl(), create_tl_block_id(block_id), collated_data_file_hash));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  CandidateShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  PublicKey source;
+  BlockIdExt block_id;
+  FileHash collated_data_file_hash;
+};
+
+class BlockInfoShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class BlockInfo {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_blockInfo>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_blockInfo>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  BlockInfoShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+};  // namespace fileref
+
+class FileReferenceShort {
+ private:
+  td::Variant<fileref::Empty, fileref::BlockShort, fileref::ZeroStateShort, fileref::PersistentStateShort,
+              fileref::ProofShort, fileref::ProofShort, fileref::ProofLinkShort, fileref::SignaturesShort,
+              fileref::CandidateShort, fileref::BlockInfoShort>
+      ref_;
+
+ public:
+  template <typename T>
+  FileReferenceShort(T x) : ref_(std::move(x)) {
+  }
+  FileReferenceShort() : ref_(fileref::Empty{}) {
+  }
+
+  static td::Result<FileReferenceShort> create(std::string filename);
+
+  auto &ref() {
+    return ref_;
+  }
+
+  td::Bits256 hash() const;
+  ShardIdFull shard() const;
+  std::string filename_short() const;
+};
+
+class FileReference {
+ private:
+  td::Variant<fileref::Empty, fileref::Block, fileref::ZeroState, fileref::PersistentState, fileref::Proof,
+              fileref::Proof, fileref::ProofLink, fileref::Signatures, fileref::Candidate, fileref::BlockInfo>
+      ref_;
+
+ public:
+  template <typename T>
+  FileReference(T x) : ref_(std::move(x)) {
+  }
+  FileReference() : ref_(fileref::Empty{}) {
+  }
+  FileReference(tl_object_ptr<ton_api::db_filedb_Key> key);
+
+  static td::Result<FileReference> create(std::string filename);
+
+  auto &ref() {
+    return ref_;
+  }
+
+  FileReferenceShort shortref() const;
+
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const;
+  td::Bits256 hash() const;
+  ShardIdFull shard() const;
+  std::string filename() const;
+  std::string filename_short() const;
+};
+
+}  // namespace validator
+
+}  // namespace ton
+

validator/db/ltdb.cpp

@@ -1,310 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#include "ltdb.hpp"
-#include "rootdb.hpp"
-#include "auto/tl/ton_api.h"
-#include "ton/ton-tl.hpp"
-#include "ton/ton-io.hpp"
-#include "td/db/RocksDb.h"
-
-namespace ton {
-
-namespace validator {
-
-void LtDb::add_new_block(BlockIdExt block_id, LogicalTime lt, UnixTime ts, td::Promise<td::Unit> promise) {
-  auto key = get_desc_key(block_id.shard_full());
-
-  std::string value;
-  auto R = kv_->get(key.as_slice(), value);
-  R.ensure();
-  tl_object_ptr<ton_api::db_lt_desc_value> v;
-  bool add_shard = false;
-  if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
-    auto F = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
-    F.ensure();
-    v = F.move_as_ok();
-  } else {
-    v = create_tl_object<ton_api::db_lt_desc_value>(1, 1, 0, 0, 0);
-    add_shard = true;
-  }
-  if (block_id.id.seqno <= static_cast<td::uint32>(v->last_seqno_) || lt <= static_cast<LogicalTime>(v->last_lt_) ||
-      ts <= static_cast<UnixTime>(v->last_ts_)) {
-    promise.set_value(td::Unit());
-    return;
-  }
-  auto db_value = create_serialize_tl_object<ton_api::db_lt_el_value>(create_tl_block_id(block_id), lt, ts);
-  auto db_key = get_el_key(block_id.shard_full(), v->last_idx_++);
-  auto status_key = create_serialize_tl_object<ton_api::db_lt_status_key>();
-  v->last_seqno_ = block_id.id.seqno;
-  v->last_lt_ = lt;
-  v->last_ts_ = ts;
-
-  td::uint32 idx = 0;
-  if (add_shard) {
-    auto G = kv_->get(status_key.as_slice(), value);
-    G.ensure();
-    if (G.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-      idx = 0;
-    } else {
-      auto F = fetch_tl_object<ton_api::db_lt_status_value>(value, true);
-      F.ensure();
-      auto f = F.move_as_ok();
-      idx = f->total_shards_;
-    }
-  }
-
-  kv_->begin_transaction().ensure();
-  kv_->set(key, serialize_tl_object(v, true)).ensure();
-  kv_->set(db_key, db_value.as_slice()).ensure();
-  if (add_shard) {
-    auto shard_key = create_serialize_tl_object<ton_api::db_lt_shard_key>(idx);
-    auto shard_value = create_serialize_tl_object<ton_api::db_lt_shard_value>(block_id.id.workchain, block_id.id.shard);
-    kv_->set(status_key.as_slice(), create_serialize_tl_object<ton_api::db_lt_status_value>(idx + 1)).ensure();
-    kv_->set(shard_key.as_slice(), shard_value.as_slice()).ensure();
-  }
-  kv_->commit_transaction().ensure();
-
-  promise.set_value(td::Unit());
-}
-
-void LtDb::get_block_common(AccountIdPrefixFull account_id,
-                            std::function<td::int32(ton_api::db_lt_desc_value &)> compare_desc,
-                            std::function<td::int32(ton_api::db_lt_el_value &)> compare, bool exact,
-                            td::Promise<BlockIdExt> promise) {
-  bool f = false;
-  BlockIdExt block_id;
-  td::uint32 ls = 0;
-  for (td::uint32 len = 0; len <= 60; len++) {
-    auto s = shard_prefix(account_id, len);
-    auto key = get_desc_key(s);
-    std::string value;
-    auto F = kv_->get(key, value);
-    F.ensure();
-    if (F.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-      if (!f) {
-        continue;
-      } else {
-        break;
-      }
-    }
-    f = true;
-    auto G = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
-    G.ensure();
-    auto g = G.move_as_ok();
-    if (compare_desc(*g.get()) > 0) {
-      continue;
-    }
-    td::uint32 l = g->first_idx_ - 1;
-    BlockIdExt lseq;
-    td::uint32 r = g->last_idx_;
-    BlockIdExt rseq;
-    while (r - l > 1) {
-      auto x = (r + l) / 2;
-      auto db_key = get_el_key(s, x);
-      F = kv_->get(db_key, value);
-      F.ensure();
-      CHECK(F.move_as_ok() == td::KeyValue::GetStatus::Ok);
-      auto E = fetch_tl_object<ton_api::db_lt_el_value>(td::BufferSlice{value}, true);
-      E.ensure();
-      auto e = E.move_as_ok();
-      int cmp_val = compare(*e.get());
-
-      if (cmp_val < 0) {
-        rseq = create_block_id(e->id_);
-        r = x;
-      } else if (cmp_val > 0) {
-        lseq = create_block_id(e->id_);
-        l = x;
-      } else {
-        promise.set_value(create_block_id(e->id_));
-        return;
-      }
-    }
-    if (rseq.is_valid()) {
-      if (!block_id.is_valid()) {
-        block_id = rseq;
-      } else if (block_id.id.seqno > rseq.id.seqno) {
-        block_id = rseq;
-      }
-    }
-    if (lseq.is_valid()) {
-      if (ls < lseq.id.seqno) {
-        ls = lseq.id.seqno;
-      }
-    }
-    if (block_id.is_valid() && ls + 1 == block_id.id.seqno) {
-      if (!exact) {
-        promise.set_value(std::move(block_id));
-      } else {
-        promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found"));
-      }
-      return;
-    }
-  }
-  if (!exact && block_id.is_valid()) {
-    promise.set_value(std::move(block_id));
-  } else {
-    promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found"));
-  }
-}
-
-void LtDb::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockIdExt> promise) {
-  return get_block_common(
-      account_id,
-      [lt](ton_api::db_lt_desc_value &w) {
-        return lt > static_cast<LogicalTime>(w.last_lt_) ? 1 : lt == static_cast<LogicalTime>(w.last_lt_) ? 0 : -1;
-      },
-      [lt](ton_api::db_lt_el_value &w) {
-        return lt > static_cast<LogicalTime>(w.lt_) ? 1 : lt == static_cast<LogicalTime>(w.lt_) ? 0 : -1;
-      },
-      false, std::move(promise));
-}
-
-void LtDb::get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise<BlockIdExt> promise) {
-  return get_block_common(
-      account_id,
-      [seqno](ton_api::db_lt_desc_value &w) {
-        return seqno > static_cast<BlockSeqno>(w.last_seqno_)
-                   ? 1
-                   : seqno == static_cast<BlockSeqno>(w.last_seqno_) ? 0 : -1;
-      },
-      [seqno](ton_api::db_lt_el_value &w) {
-        return seqno > static_cast<BlockSeqno>(w.id_->seqno_)
-                   ? 1
-                   : seqno == static_cast<BlockSeqno>(w.id_->seqno_) ? 0 : -1;
-      },
-      true, std::move(promise));
-}
-
-void LtDb::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise<BlockIdExt> promise) {
-  return get_block_common(
-      account_id,
-      [ts](ton_api::db_lt_desc_value &w) {
-        return ts > static_cast<UnixTime>(w.last_ts_) ? 1 : ts == static_cast<UnixTime>(w.last_ts_) ? 0 : -1;
-      },
-      [ts](ton_api::db_lt_el_value &w) {
-        return ts > static_cast<UnixTime>(w.ts_) ? 1 : ts == static_cast<UnixTime>(w.ts_) ? 0 : -1;
-      },
-      false, std::move(promise));
-}
-
-td::BufferSlice LtDb::get_desc_key(ShardIdFull shard) {
-  return create_serialize_tl_object<ton_api::db_lt_desc_key>(shard.workchain, shard.shard);
-}
-
-td::BufferSlice LtDb::get_el_key(ShardIdFull shard, td::uint32 idx) {
-  return create_serialize_tl_object<ton_api::db_lt_el_key>(shard.workchain, shard.shard, idx);
-}
-
-void LtDb::start_up() {
-  kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
-}
-
-void LtDb::truncate_workchain(ShardIdFull shard, td::Ref<MasterchainState> state) {
-  auto key = get_desc_key(shard);
-  std::string value;
-  auto R = kv_->get(key, value);
-  R.ensure();
-  CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok);
-  auto F = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
-  F.ensure();
-  auto f = F.move_as_ok();
-
-  auto shards = state->get_shards();
-  BlockSeqno seqno = 0;
-  if (shard.is_masterchain()) {
-    seqno = state->get_seqno();
-  } else {
-    for (auto s : shards) {
-      if (shard_intersects(s->shard(), shard)) {
-        seqno = s->top_block_id().seqno();
-        break;
-      }
-    }
-  }
-
-  while (f->last_idx_ > f->first_idx_) {
-    auto db_key = get_el_key(shard, f->last_idx_ - 1);
-    R = kv_->get(db_key, value);
-    R.ensure();
-    CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok);
-    auto E = fetch_tl_object<ton_api::db_lt_el_value>(td::BufferSlice{value}, true);
-    E.ensure();
-    auto e = E.move_as_ok();
-
-    bool to_delete = static_cast<td::uint32>(e->id_->seqno_) > seqno;
-
-    if (!to_delete) {
-      break;
-    } else {
-      f->last_idx_--;
-      kv_->erase(db_key).ensure();
-    }
-  }
-
-  if (f->first_idx_ == f->last_idx_) {
-    f->last_ts_ = 0;
-    f->last_lt_ = 0;
-    f->last_seqno_ = 0;
-  }
-
-  kv_->set(key, serialize_tl_object(f, true)).ensure();
-}
-
-void LtDb::truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) {
-  auto status_key = create_serialize_tl_object<ton_api::db_lt_status_key>();
-  td::Result<td::KeyValue::GetStatus> R;
-  td::uint32 total_shards = 0;
-  {
-    std::string value;
-    R = kv_->get(status_key.as_slice(), value);
-    R.ensure();
-    if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-      promise.set_value(td::Unit());
-      return;
-    }
-    auto F = fetch_tl_object<ton_api::db_lt_status_value>(value, true);
-    F.ensure();
-    auto f = F.move_as_ok();
-    total_shards = f->total_shards_;
-    if (total_shards == 0) {
-      promise.set_value(td::Unit());
-      return;
-    }
-  }
-  kv_->begin_transaction().ensure();
-  for (td::uint32 idx = 0; idx < total_shards; idx++) {
-    auto shard_key = create_serialize_tl_object<ton_api::db_lt_shard_key>(idx);
-    std::string value;
-    R = kv_->get(shard_key.as_slice(), value);
-    R.ensure();
-    CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok);
-    auto F = fetch_tl_object<ton_api::db_lt_shard_value>(value, true);
-    F.ensure();
-    auto f = F.move_as_ok();
-
-    truncate_workchain(ShardIdFull{f->workchain_, static_cast<td::uint64>(f->shard_)}, state);
-  }
-  kv_->commit_transaction().ensure();
-  promise.set_value(td::Unit());
-}
-
-}  // namespace validator
-
-}  // namespace ton

validator/db/ltdb.hpp

@@ -1,67 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#pragma once
-
-#include "td/actor/actor.h"
-#include "td/db/KeyValueAsync.h"
-#include "validator/interfaces/db.h"
-
-#include "ton/ton-types.h"
-
-#include "auto/tl/ton_api.h"
-
-namespace ton {
-
-namespace validator {
-
-class RootDb;
-
-class LtDb : public td::actor::Actor {
- public:
-  void add_new_block(BlockIdExt block_id, LogicalTime lt, UnixTime ts, td::Promise<td::Unit> promise);
-  void get_block_common(AccountIdPrefixFull account_id,
-                        std::function<td::int32(ton_api::db_lt_desc_value &)> compare_desc,
-                        std::function<td::int32(ton_api::db_lt_el_value &)> compare, bool exact,
-                        td::Promise<BlockIdExt> promise);
-  void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockIdExt> promise);
-  void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise<BlockIdExt> promise);
-  void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise<BlockIdExt> promise);
-
-  void truncate_workchain(ShardIdFull shard, td::Ref<MasterchainState> state);
-  void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise);
-
-  void start_up() override;
-
-  LtDb(td::actor::ActorId<RootDb> root_db, std::string db_path) : root_db_(root_db), db_path_(std::move(db_path)) {
-  }
-
- private:
-  td::BufferSlice get_desc_key(ShardIdFull shard);
-  td::BufferSlice get_el_key(ShardIdFull shard, td::uint32 idx);
-  td::BufferSlice get_from_db(ShardIdFull shard, td::uint32 idx);
-
-  std::shared_ptr<td::KeyValue> kv_;
-
-  td::actor::ActorId<RootDb> root_db_;
-  std::string db_path_;
-};
-
-}  // namespace validator
-
-}  // namespace ton

validator/db/package.cpp

@@ -34,7 +34,7 @@ td::Status Package::truncate(td::uint64 size) {
   return fd_.truncate_to_current_position(size + header_size());
 }
 
-td::uint64 Package::append(std::string filename, td::Slice data) {
+td::uint64 Package::append(std::string filename, td::Slice data, bool sync) {
   CHECK(data.size() <= max_data_size());
   CHECK(filename.size() <= max_filename_size());
   auto size = fd_.get_size().move_as_ok();
@@ -48,10 +48,16 @@ td::uint64 Package::append(std::string filename, td::Slice data) {
   size += filename.size();
   CHECK(fd_.pwrite(data, size).move_as_ok() == data.size());
   size += data.size();
-  fd_.sync().ensure();
+  if (sync) {
+    fd_.sync().ensure();
+  }
   return orig_size - header_size();
 }
 
+void Package::sync() {
+  fd_.sync().ensure();
+}
+
 td::uint64 Package::size() const {
   return fd_.get_size().move_as_ok() - header_size();
 }
@@ -140,4 +146,28 @@ td::Result<Package> Package::open(std::string path, bool read_only, bool create)
   return Package{std::move(fd)};
 }
 
+void Package::iterate(std::function<bool(std::string, td::BufferSlice, td::uint64)> func) {
+  td::uint64 p = 0;
+
+  td::uint64 size = fd_.get_size().move_as_ok();
+  if (size < header_size()) {
+    LOG(ERROR) << "too short archive";
+    return;
+  }
+  size -= header_size();
+  while (p != size) {
+    auto R = read(p);
+    if (R.is_error()) {
+      LOG(ERROR) << "broken archive: " << R.move_as_error();
+      return;
+    }
+    auto q = R.move_as_ok();
+    if (!func(q.first, q.second.clone(), p)) {
+      break;
+    }
+
+    p = advance(p).move_as_ok();
+  }
+}
+
 }  // namespace ton

validator/db/package.hpp

@@ -14,24 +14,17 @@ class Package {
 
   td::Status truncate(td::uint64 size);
 
-  td::uint64 append(std::string filename, td::Slice data);
+  td::uint64 append(std::string filename, td::Slice data, bool sync = true);
+  void sync();
   td::uint64 size() const;
   td::Result<std::pair<std::string, td::BufferSlice>> read(td::uint64 offset) const;
+
   td::Result<td::uint64> advance(td::uint64 offset);
+  void iterate(std::function<bool(std::string, td::BufferSlice, td::uint64)> func);
 
-  struct Iterator {
-    td::uint64 offset;
-    Package &package;
-
-    Iterator operator++(int);
-    const Iterator operator++(int) const;
-    td::Result<std::pair<std::string, td::BufferSlice>> read() const;
-  };
-
-  Iterator begin();
-  const Iterator begin() const;
-  Iterator end();
-  const Iterator end() const;
+  td::FileFd &fd() {
+    return fd_;
+  }
 
  private:
   td::FileFd fd_;

validator/db/rootdb.cpp

@@ -32,22 +32,22 @@ namespace ton {
 namespace validator {
 
 void RootDb::store_block_data(BlockHandle handle, td::Ref<BlockData> block, td::Promise<td::Unit> promise) {
-  if (handle->moved_to_storage() || handle->moved_to_archive()) {
+  if (handle->received()) {
     promise.set_value(td::Unit());
     return;
   }
-  auto id = block_db_.get();
 
-  auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      handle->set_received();
-      td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise));
-    }
-  });
+  auto P = td::PromiseCreator::lambda(
+      [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          handle->set_received();
+          td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise));
+        }
+      });
 
-  td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::Block{handle->id()}}, block->data(),
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_file, handle, fileref::Block{handle->id()}, block->data(),
                           std::move(P));
 }
 
@@ -64,45 +64,34 @@ void RootDb::get_block_data(BlockHandle handle, td::Promise<td::Ref<BlockData>>
           }
         });
 
-    if (handle->moved_to_archive()) {
-      td::actor::send_closure(new_archive_db_, &ArchiveManager::read, handle->unix_time(), handle->is_key_block(),
-                              FileDb::RefId{fileref::Block{handle->id()}}, std::move(P));
-    } else {
-      td::actor::send_closure(handle->moved_to_storage() ? old_archive_db_.get() : file_db_.get(), &FileDb::load_file,
-                              FileDb::RefId{fileref::Block{handle->id()}}, std::move(P));
-    }
+    td::actor::send_closure(archive_db_, &ArchiveManager::get_file, handle, fileref::Block{handle->id()}, std::move(P));
   }
 }
 
 void RootDb::store_block_signatures(BlockHandle handle, td::Ref<BlockSignatureSet> data,
                                     td::Promise<td::Unit> promise) {
-  if (handle->moved_to_storage() || handle->moved_to_archive()) {
+  if (handle->inited_signatures() || handle->moved_to_archive()) {
     promise.set_value(td::Unit());
     return;
   }
-  auto id = block_db_.get();
 
-  auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      handle->set_signatures();
-      td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise));
-    }
-  });
-
-  td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::Signatures{handle->id()}},
+  auto P = td::PromiseCreator::lambda(
+      [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          handle->set_signatures();
+          td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise));
+        }
+      });
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_temp_file_short, fileref::Signatures{handle->id()},
                           data->serialize(), std::move(P));
 }
 
 void RootDb::get_block_signatures(BlockHandle handle, td::Promise<td::Ref<BlockSignatureSet>> promise) {
-  if (!handle->inited_signatures()) {
+  if (!handle->inited_signatures() || handle->moved_to_archive()) {
     promise.set_error(td::Status::Error(ErrorCode::notready, "not in db"));
   } else {
-    if (handle->moved_to_storage() || handle->moved_to_archive()) {
-      promise.set_error(td::Status::Error(ErrorCode::error, "signatures already gc'd"));
-      return;
-    }
     auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
       if (R.is_error()) {
         promise.set_error(R.move_as_error());
@@ -110,28 +99,28 @@ void RootDb::get_block_signatures(BlockHandle handle, td::Promise<td::Ref<BlockS
         promise.set_result(create_signature_set(R.move_as_ok()));
       }
     });
-    td::actor::send_closure(file_db_, &FileDb::load_file, FileDb::RefId{fileref::Signatures{handle->id()}},
+    td::actor::send_closure(archive_db_, &ArchiveManager::get_temp_file_short, fileref::Signatures{handle->id()},
                             std::move(P));
   }
 }
 
 void RootDb::store_block_proof(BlockHandle handle, td::Ref<Proof> proof, td::Promise<td::Unit> promise) {
-  if (handle->moved_to_storage() || handle->moved_to_archive()) {
+  if (handle->inited_proof()) {
     promise.set_value(td::Unit());
     return;
   }
-  auto id = block_db_.get();
 
-  auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      handle->set_proof();
-      td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise));
-    }
-  });
+  auto P = td::PromiseCreator::lambda(
+      [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          handle->set_proof();
+          td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise));
+        }
+      });
 
-  td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::Proof{handle->id()}}, proof->data(),
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_file, handle, fileref::Proof{handle->id()}, proof->data(),
                           std::move(P));
 }
 
@@ -147,34 +136,27 @@ void RootDb::get_block_proof(BlockHandle handle, td::Promise<td::Ref<Proof>> pro
             promise.set_result(create_proof(id, R.move_as_ok()));
           }
         });
-    if (handle->moved_to_archive()) {
-      td::actor::send_closure(new_archive_db_, &ArchiveManager::read, handle->unix_time(), handle->is_key_block(),
-                              FileDb::RefId{fileref::Proof{handle->id()}}, std::move(P));
-    } else {
-      td::actor::send_closure(handle->moved_to_storage() ? old_archive_db_.get() : file_db_.get(), &FileDb::load_file,
-                              FileDb::RefId{fileref::Proof{handle->id()}}, std::move(P));
-    }
+    td::actor::send_closure(archive_db_, &ArchiveManager::get_file, handle, fileref::Proof{handle->id()}, std::move(P));
   }
 }
 
 void RootDb::store_block_proof_link(BlockHandle handle, td::Ref<ProofLink> proof, td::Promise<td::Unit> promise) {
-  if (handle->moved_to_storage() || handle->moved_to_archive()) {
+  if (handle->inited_proof_link()) {
     promise.set_value(td::Unit());
     return;
   }
-  auto id = block_db_.get();
 
-  auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      handle->set_proof_link();
-      td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise));
-    }
-  });
-
-  td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::ProofLink{handle->id()}}, proof->data(),
-                          std::move(P));
+  auto P = td::PromiseCreator::lambda(
+      [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          handle->set_proof_link();
+          td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise));
+        }
+      });
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_file, handle, fileref::ProofLink{handle->id()},
+                          proof->data(), std::move(P));
 }
 
 void RootDb::get_block_proof_link(BlockHandle handle, td::Promise<td::Ref<ProofLink>> promise) {
@@ -189,13 +171,8 @@ void RootDb::get_block_proof_link(BlockHandle handle, td::Promise<td::Ref<ProofL
             promise.set_result(create_proof_link(id, R.move_as_ok()));
           }
         });
-    if (handle->moved_to_archive()) {
-      td::actor::send_closure(new_archive_db_, &ArchiveManager::read, handle->unix_time(), handle->is_key_block(),
-                              FileDb::RefId{fileref::ProofLink{handle->id()}}, std::move(P));
-    } else {
-      td::actor::send_closure(handle->moved_to_storage() ? old_archive_db_.get() : file_db_.get(), &FileDb::load_file,
-                              FileDb::RefId{fileref::ProofLink{handle->id()}}, std::move(P));
-    }
+    td::actor::send_closure(archive_db_, &ArchiveManager::get_file, handle, fileref::ProofLink{handle->id()},
+                            std::move(P));
   }
 }
 
@@ -204,16 +181,16 @@ void RootDb::store_block_candidate(BlockCandidate candidate, td::Promise<td::Uni
       PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}.tl(), create_tl_block_id(candidate.id),
       std::move(candidate.data), std::move(candidate.collated_data));
 
-  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<FileHash> R) mutable {
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Unit> R) mutable {
     if (R.is_error()) {
       promise.set_error(R.move_as_error());
     } else {
       promise.set_value(td::Unit());
     }
   });
-  td::actor::send_closure(file_db_, &FileDb::store_file,
-                          FileDb::RefId{fileref::Candidate{PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}},
-                                                           candidate.id, candidate.collated_file_hash}},
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_temp_file_short,
+                          fileref::Candidate{PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.id,
+                                             candidate.collated_file_hash},
                           std::move(obj), std::move(P));
 }
 
@@ -234,18 +211,18 @@ void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash colla
                                        std::move(val->collated_data_)});
     }
   });
-  td::actor::send_closure(file_db_, &FileDb::load_file,
-                          FileDb::RefId{fileref::Candidate{source, id, collated_data_file_hash}}, std::move(P));
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_temp_file_short,
+                          fileref::Candidate{source, id, collated_data_file_hash}, std::move(P));
 }
 
 void RootDb::store_block_state(BlockHandle handle, td::Ref<ShardState> state,
                                td::Promise<td::Ref<ShardState>> promise) {
-  if (handle->moved_to_storage() || handle->moved_to_archive()) {
+  if (handle->moved_to_archive()) {
     promise.set_value(std::move(state));
     return;
   }
   if (!handle->inited_state_boc()) {
-    auto P = td::PromiseCreator::lambda([b = block_db_.get(), root_hash = state->root_hash(), handle,
+    auto P = td::PromiseCreator::lambda([b = archive_db_.get(), root_hash = state->root_hash(), handle,
                                          promise = std::move(promise)](td::Result<td::Ref<vm::DataCell>> R) mutable {
       if (R.is_error()) {
         promise.set_error(R.move_as_error());
@@ -262,7 +239,7 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref<ShardState> state,
               promise.set_value(std::move(state));
             });
 
-        td::actor::send_closure(b, &BlockDb::store_block_handle, std::move(handle), std::move(P));
+        td::actor::send_closure(b, &ArchiveManager::update_handle, std::move(handle), std::move(P));
       }
     });
     td::actor::send_closure(cell_db_, &CellDb::store_cell, handle->id(), state->root_cell(), std::move(P));
@@ -295,83 +272,46 @@ void RootDb::get_block_state(BlockHandle handle, td::Promise<td::Ref<ShardState>
 
 void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
                                          td::Promise<td::Unit> promise) {
-  auto id = block_db_.get();
-
-  auto P = td::PromiseCreator::lambda([id, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      promise.set_value(td::Unit());
-    }
-  });
-
-  td::actor::send_closure(old_archive_db_, &FileDb::store_file,
-                          FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, std::move(state),
-                          std::move(P));
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id,
+                          std::move(state), std::move(promise));
 }
 
 void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id,
                                        td::Promise<td::BufferSlice> promise) {
-  td::actor::send_closure(old_archive_db_, &FileDb::load_file,
-                          FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, std::move(promise));
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id,
+                          std::move(promise));
 }
 
 void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,
                                              td::int64 max_size, td::Promise<td::BufferSlice> promise) {
-  td::actor::send_closure(old_archive_db_, &FileDb::load_file_slice,
-                          FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, offset, max_size,
-                          std::move(promise));
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_slice, block_id, masterchain_block_id,
+                          offset, max_size, std::move(promise));
 }
 
 void RootDb::check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id,
                                                 td::Promise<bool> promise) {
-  td::actor::send_closure(old_archive_db_, &FileDb::check_file,
-                          FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, std::move(promise));
+  td::actor::send_closure(archive_db_, &ArchiveManager::check_persistent_state, block_id, masterchain_block_id,
+                          std::move(promise));
 }
 
 void RootDb::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise<td::Unit> promise) {
-  auto id = block_db_.get();
-
-  auto P = td::PromiseCreator::lambda([id, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      promise.set_value(td::Unit());
-    }
-  });
-
-  td::actor::send_closure(old_archive_db_, &FileDb::store_file, FileDb::RefId{fileref::ZeroState{block_id}},
-                          std::move(state), std::move(P));
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_zero_state, block_id, std::move(state), std::move(promise));
 }
 
 void RootDb::get_zero_state_file(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {
-  td::actor::send_closure(old_archive_db_, &FileDb::load_file, FileDb::RefId{fileref::ZeroState{block_id}},
-                          std::move(promise));
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_zero_state, block_id, std::move(promise));
 }
 
 void RootDb::check_zero_state_file_exists(BlockIdExt block_id, td::Promise<bool> promise) {
-  td::actor::send_closure(old_archive_db_, &FileDb::check_file, FileDb::RefId{fileref::ZeroState{block_id}},
-                          std::move(promise));
+  td::actor::send_closure(archive_db_, &ArchiveManager::check_zero_state, block_id, std::move(promise));
 }
 
 void RootDb::store_block_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
-  if (handle->moved_to_archive()) {
-    td::actor::send_closure(new_archive_db_, &ArchiveManager::write_handle, std::move(handle), std::move(promise));
-  } else {
-    td::actor::send_closure(block_db_, &BlockDb::store_block_handle, std::move(handle), std::move(promise));
-  }
+  td::actor::send_closure(archive_db_, &ArchiveManager::update_handle, std::move(handle), std::move(promise));
 }
 
 void RootDb::get_block_handle(BlockIdExt id, td::Promise<BlockHandle> promise) {
-  auto P = td::PromiseCreator::lambda(
-      [db = block_db_.get(), id, promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
-        if (R.is_error()) {
-          td::actor::send_closure(db, &BlockDb::get_block_handle, id, std::move(promise));
-        } else {
-          promise.set_value(R.move_as_ok());
-        }
-      });
-  td::actor::send_closure(new_archive_db_, &ArchiveManager::read_handle, id, std::move(P));
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_handle, id, std::move(promise));
 }
 
 void RootDb::try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice> promise) {
@@ -379,24 +319,20 @@ void RootDb::try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice
 }
 
 void RootDb::apply_block(BlockHandle handle, td::Promise<td::Unit> promise) {
-  if (handle->id().id.seqno == 0) {
-    promise.set_value(td::Unit());
-  } else {
-    td::actor::send_closure(lt_db_, &LtDb::add_new_block, handle->id(), handle->logical_time(), handle->unix_time(),
-                            std::move(promise));
-  }
+  td::actor::create_actor<BlockArchiver>("archiver", std::move(handle), archive_db_.get(), std::move(promise))
+      .release();
 }
 
-void RootDb::get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockIdExt> promise) {
-  td::actor::send_closure(lt_db_, &LtDb::get_block_by_lt, account, lt, std::move(promise));
+void RootDb::get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockHandle> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_block_by_lt, account, lt, std::move(promise));
 }
 
-void RootDb::get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockIdExt> promise) {
-  td::actor::send_closure(lt_db_, &LtDb::get_block_by_unix_time, account, ts, std::move(promise));
+void RootDb::get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockHandle> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_block_by_unix_time, account, ts, std::move(promise));
 }
 
-void RootDb::get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockIdExt> promise) {
-  td::actor::send_closure(lt_db_, &LtDb::get_block_by_seqno, account, seqno, std::move(promise));
+void RootDb::get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockHandle> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_block_by_seqno, account, seqno, std::move(promise));
 }
 
 void RootDb::update_init_masterchain_block(BlockIdExt block, td::Promise<td::Unit> promise) {
@@ -451,19 +387,13 @@ void RootDb::get_hardforks(td::Promise<std::vector<BlockIdExt>> promise) {
 
 void RootDb::start_up() {
   cell_db_ = td::actor::create_actor<CellDb>("celldb", actor_id(this), root_path_ + "/celldb/");
-  block_db_ = td::actor::create_actor<BlockDb>("blockdb", actor_id(this), root_path_ + "/blockdb/");
-  file_db_ = td::actor::create_actor<FileDb>("filedb", actor_id(this), root_path_ + "/files/", depth_, false);
-  old_archive_db_ =
-      td::actor::create_actor<FileDb>("filedbarchive", actor_id(this), root_path_ + "/archive/", depth_, true);
-  lt_db_ = td::actor::create_actor<LtDb>("ltdb", actor_id(this), root_path_ + "/ltdb/");
   state_db_ = td::actor::create_actor<StateDb>("statedb", actor_id(this), root_path_ + "/state/");
   static_files_db_ = td::actor::create_actor<StaticFilesDb>("staticfilesdb", actor_id(this), root_path_ + "/static/");
-  new_archive_db_ = td::actor::create_actor<ArchiveManager>("archivemanager", root_path_ + "/archive/");
+  archive_db_ = td::actor::create_actor<ArchiveManager>("archive", actor_id(this), root_path_);
 }
 
-void RootDb::archive(BlockIdExt block_id, td::Promise<td::Unit> promise) {
-  td::actor::create_actor<BlockArchiver>("archiveblock", block_id, actor_id(this), file_db_.get(),
-                                         old_archive_db_.get(), new_archive_db_.get(), std::move(promise))
+void RootDb::archive(BlockHandle handle, td::Promise<td::Unit> promise) {
+  td::actor::create_actor<BlockArchiver>("archiveblock", std::move(handle), archive_db_.get(), std::move(promise))
       .release();
 }
 
@@ -475,57 +405,86 @@ void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise<bool> promise) {
   td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_info_gc, block_id, std::move(promise));
 }
 
-void RootDb::allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise<bool> promise) {
-  ref_id.visit(
-      td::overloaded([&](const fileref::Empty &key) { UNREACHABLE(); },
-                     [&](const fileref::Block &key) {
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_data_gc, key.block_id,
-                                               is_archive, std::move(promise));
-                     },
-                     [&](const fileref::ZeroState &key) {
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_zero_state_file_gc,
-                                               key.block_id, std::move(promise));
-                     },
-                     [&](const fileref::PersistentState &key) {
-                       CHECK(is_archive);
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_persistent_state_file_gc,
-                                               key.block_id, key.masterchain_block_id, std::move(promise));
-                     },
-                     [&](const fileref::Proof &key) {
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_proof_gc,
-                                               key.block_id, is_archive, std::move(promise));
-                     },
-                     [&](const fileref::ProofLink &key) {
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_proof_link_gc,
-                                               key.block_id, is_archive, std::move(promise));
-                     },
-                     [&](const fileref::Signatures &key) {
-                       CHECK(!is_archive);
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_signatures_gc,
-                                               key.block_id, std::move(promise));
-                     },
-                     [&](const fileref::Candidate &key) {
-                       CHECK(!is_archive);
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_candidate_gc,
-                                               key.block_id, std::move(promise));
-                     },
-                     [&](const fileref::BlockInfo &key) { UNREACHABLE(); }));
-}
-
 void RootDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
   auto merger = StatsMerger::create(std::move(promise));
-
-  td::actor::send_closure(file_db_, &FileDb::prepare_stats, merger.make_promise("filedb."));
-  td::actor::send_closure(old_archive_db_, &FileDb::prepare_stats, merger.make_promise("archivedb."));
 }
 
 void RootDb::truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) {
   td::MultiPromise mp;
   auto ig = mp.init_guard();
   ig.add_promise(std::move(promise));
+}
 
-  td::actor::send_closure(lt_db_, &LtDb::truncate, state, ig.get_promise());
-  td::actor::send_closure(block_db_, &BlockDb::truncate, state, ig.get_promise());
+void RootDb::add_key_block_proof(td::Ref<Proof> proof, td::Promise<td::Unit> promise) {
+  auto i = proof->get_basic_header_info().move_as_ok();
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_key_block_proof, i.utime, proof->block_id().seqno(),
+                          i.end_lt, fileref::Proof{proof->block_id()}, proof->data(), std::move(promise));
+}
+
+void RootDb::add_key_block_proof_link(td::Ref<ProofLink> proof, td::Promise<td::Unit> promise) {
+  auto i = proof->get_basic_header_info().move_as_ok();
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_key_block_proof, i.utime, proof->block_id().seqno(),
+                          i.end_lt, fileref::ProofLink{proof->block_id()}, proof->data(), std::move(promise));
+}
+void RootDb::get_key_block_proof(BlockIdExt block_id, td::Promise<td::Ref<Proof>> promise) {
+  auto P = td::PromiseCreator::lambda([block_id, promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_error()) {
+      promise.set_error(R.move_as_error());
+    } else {
+      promise.set_result(create_proof(block_id, R.move_as_ok()));
+    }
+  });
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::Proof{block_id}, std::move(P));
+}
+void RootDb::get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::Ref<ProofLink>> promise) {
+  auto P = td::PromiseCreator::lambda([block_id, promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_error()) {
+      promise.set_error(R.move_as_error());
+    } else {
+      promise.set_result(create_proof_link(block_id, R.move_as_ok()));
+    }
+  });
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::Proof{block_id}, std::move(P));
+}
+
+void RootDb::check_key_block_proof_exists(BlockIdExt block_id, td::Promise<bool> promise) {
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_error()) {
+      promise.set_result(false);
+    } else {
+      promise.set_result(true);
+    }
+  });
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::Proof{block_id}, std::move(P));
+}
+void RootDb::check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise<bool> promise) {
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_error()) {
+      promise.set_result(false);
+    } else {
+      promise.set_result(true);
+    }
+  });
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::ProofLink{block_id},
+                          std::move(P));
+}
+
+void RootDb::get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_id, masterchain_seqno, std::move(promise));
+}
+
+void RootDb::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                               td::Promise<td::BufferSlice> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_slice, archive_id, offset, limit,
+                          std::move(promise));
+}
+
+void RootDb::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise));
+}
+
+void RootDb::run_gc(UnixTime ts) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, ts);
 }
 
 }  // namespace validator

validator/db/rootdb.hpp

@@ -22,13 +22,10 @@
 #include "td/db/KeyValueAsync.h"
 #include "ton/ton-types.h"
 
-#include "blockdb.hpp"
 #include "celldb.hpp"
-#include "filedb.hpp"
-#include "ltdb.hpp"
 #include "statedb.hpp"
 #include "staticfilesdb.hpp"
-#include "archive-db.hpp"
+#include "archive-manager.hpp"
 
 namespace ton {
 
@@ -85,9 +82,9 @@ class RootDb : public Db {
   void try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice> promise) override;
 
   void apply_block(BlockHandle handle, td::Promise<td::Unit> promise) override;
-  void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockIdExt> promise) override;
-  void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockIdExt> promise) override;
-  void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockIdExt> promise) override;
+  void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockHandle> promise) override;
+  void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockHandle> promise) override;
+  void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockHandle> promise) override;
 
   void update_init_masterchain_block(BlockIdExt block, td::Promise<td::Unit> promise) override;
   void get_init_masterchain_block(td::Promise<BlockIdExt> promise) override;
@@ -108,16 +105,30 @@ class RootDb : public Db {
   void update_hardforks(std::vector<BlockIdExt> blocks, td::Promise<td::Unit> promise) override;
   void get_hardforks(td::Promise<std::vector<BlockIdExt>> promise) override;
 
-  void archive(BlockIdExt block_id, td::Promise<td::Unit> promise) override;
+  void archive(BlockHandle handle, td::Promise<td::Unit> promise) override;
 
   void allow_state_gc(BlockIdExt block_id, td::Promise<bool> promise);
   void allow_block_gc(BlockIdExt block_id, td::Promise<bool> promise);
-  void allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise<bool> promise);
+  //void allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise<bool> promise);
 
   void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) override;
 
   void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) override;
 
+  void add_key_block_proof(td::Ref<Proof> proof, td::Promise<td::Unit> promise) override;
+  void add_key_block_proof_link(td::Ref<ProofLink> proof_link, td::Promise<td::Unit> promise) override;
+  void get_key_block_proof(BlockIdExt block_id, td::Promise<td::Ref<Proof>> promise) override;
+  void get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::Ref<ProofLink>> promise) override;
+  void check_key_block_proof_exists(BlockIdExt block_id, td::Promise<bool> promise) override;
+  void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise<bool> promise) override;
+
+  void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) override;
+  void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                         td::Promise<td::BufferSlice> promise) override;
+  void set_async_mode(bool mode, td::Promise<td::Unit> promise) override;
+
+  void run_gc(UnixTime ts) override;
+
  private:
   td::actor::ActorId<ValidatorManager> validator_manager_;
 
@@ -125,13 +136,9 @@ class RootDb : public Db {
   td::uint32 depth_;
 
   td::actor::ActorOwn<CellDb> cell_db_;
-  td::actor::ActorOwn<BlockDb> block_db_;
-  td::actor::ActorOwn<FileDb> file_db_;
-  td::actor::ActorOwn<FileDb> old_archive_db_;
-  td::actor::ActorOwn<LtDb> lt_db_;
   td::actor::ActorOwn<StateDb> state_db_;
   td::actor::ActorOwn<StaticFilesDb> static_files_db_;
-  td::actor::ActorOwn<ArchiveManager> new_archive_db_;
+  td::actor::ActorOwn<ArchiveManager> archive_db_;
 };
 
 }  // namespace validator

validator/db/statedb.cpp

@@ -222,6 +222,22 @@ StateDb::StateDb(td::actor::ActorId<RootDb> root_db, std::string db_path) : root
 
 void StateDb::start_up() {
   kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
+
+  std::string value;
+  auto R = kv_->get(create_serialize_tl_object<ton_api::db_state_key_dbVersion>(), value);
+  R.ensure();
+  if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+    auto F = fetch_tl_object<ton_api::db_state_dbVersion>(value, true);
+    F.ensure();
+    auto f = F.move_as_ok();
+    CHECK(f->version_ == 2);
+  } else {
+    kv_->begin_transaction().ensure();
+    kv_->set(create_serialize_tl_object<ton_api::db_state_key_dbVersion>(),
+             create_serialize_tl_object<ton_api::db_state_dbVersion>(2))
+        .ensure();
+    kv_->commit_transaction().ensure();
+  }
 }
 
 }  // namespace validator

validator/db/statedb.hpp

@@ -50,6 +50,9 @@ class StateDb : public td::actor::Actor {
   void update_hardforks(std::vector<BlockIdExt> blocks, td::Promise<td::Unit> promise);
   void get_hardforks(td::Promise<std::vector<BlockIdExt>> promise);
 
+  void update_db_version(td::uint32 version, td::Promise<td::Unit> promise);
+  void get_db_version(td::Promise<td::uint32> promise);
+
   StateDb(td::actor::ActorId<RootDb> root_db, std::string path);
 
   void start_up() override;

validator/downloaders/download-state.cpp

@@ -194,11 +194,13 @@ void DownloadShardState::written_shard_state(td::Ref<ShardState> state) {
   handle_->set_applied();
   handle_->set_split(state_->before_split());
 
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result<td::Unit> R) {
+    CHECK(handle->handle_moved_to_archive());
+    CHECK(handle->moved_to_archive())
     R.ensure();
     td::actor::send_closure(SelfId, &DownloadShardState::written_block_handle);
   });
-  handle_->flush(manager_, handle_, std::move(P));
+  td::actor::send_closure(manager_, &ValidatorManager::archive, handle_, std::move(P));
 }
 
 void DownloadShardState::written_block_handle() {

validator/fabric.h

@@ -54,8 +54,9 @@ void run_fake_accept_block_query(BlockIdExt id, td::Ref<BlockData> data, std::ve
                                  td::Promise<td::Unit> promise);
 void run_hardfork_accept_block_query(BlockIdExt id, td::Ref<BlockData> data,
                                      td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise);
-void run_apply_block_query(BlockIdExt id, td::Ref<BlockData> block, td::actor::ActorId<ValidatorManager> manager,
-                           td::Timestamp timeout, td::Promise<td::Unit> promise);
+void run_apply_block_query(BlockIdExt id, td::Ref<BlockData> block, BlockIdExt masterchain_block_id,
+                           td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
+                           td::Promise<td::Unit> promise);
 void run_check_proof_query(BlockIdExt id, td::Ref<Proof> proof, td::actor::ActorId<ValidatorManager> manager,
                            td::Timestamp timeout, td::Promise<BlockHandle> promise, bool skip_check_signatures = false);
 void run_check_proof_query(BlockIdExt id, td::Ref<Proof> proof, td::actor::ActorId<ValidatorManager> manager,

validator/full-node-master.cpp

@@ -309,6 +309,26 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo
   promise.set_value(create_serialize_tl_object<ton_api::tonNode_capabilities>(proto_version(), proto_capabilities()));
 }
 
+void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query,
+                                       td::Promise<td::BufferSlice> promise) {
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::uint64> R) mutable {
+        if (R.is_error()) {
+          promise.set_value(create_serialize_tl_object<ton_api::tonNode_archiveNotFound>());
+        } else {
+          promise.set_value(create_serialize_tl_object<ton_api::tonNode_archiveInfo>(R.move_as_ok()));
+        }
+      });
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_,
+                          std::move(P));
+}
+
+void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query,
+                                       td::Promise<td::BufferSlice> promise) {
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_,
+                          query.offset_, query.max_size_, std::move(promise));
+}
+
 void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_slave_sendExtMessage &query,
                                        td::Promise<td::BufferSlice> promise) {
   td::actor::send_closure(

validator/full-node-master.hpp

@@ -72,6 +72,10 @@ class FullNodeMasterImpl : public FullNodeMaster {
                      td::Promise<td::BufferSlice> promise);
   void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_slave_sendExtMessage &query,
                      td::Promise<td::BufferSlice> promise);
+  void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query,
+                     td::Promise<td::BufferSlice> promise);
+  void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query,
+                     td::Promise<td::BufferSlice> promise);
   // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query,
   //                   td::Promise<td::BufferSlice> promise);
   void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise<td::BufferSlice> promise);

validator/full-node-shard.cpp

@@ -31,6 +31,7 @@
 #include "net/download-state.hpp"
 #include "net/download-proof.hpp"
 #include "net/get-next-key-blocks.hpp"
+#include "net/download-archive-slice.hpp"
 
 #include "td/utils/Random.h"
 
@@ -146,6 +147,7 @@ void FullNodeShardImpl::got_next_block(td::Result<BlockHandle> R) {
 }
 
 void FullNodeShardImpl::get_next_block() {
+  //return;
   attempt_++;
   auto P = td::PromiseCreator::lambda([validator_manager = validator_manager_, attempt = attempt_,
                                        block_id = handle_->id(), SelfId = actor_id(this)](td::Result<ReceivedBlock> R) {
@@ -450,6 +452,26 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod
   promise.set_value(create_serialize_tl_object<ton_api::tonNode_capabilities>(proto_version(), proto_capabilities()));
 }
 
+void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query,
+                                      td::Promise<td::BufferSlice> promise) {
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::uint64> R) mutable {
+        if (R.is_error()) {
+          promise.set_value(create_serialize_tl_object<ton_api::tonNode_archiveNotFound>());
+        } else {
+          promise.set_value(create_serialize_tl_object<ton_api::tonNode_archiveInfo>(R.move_as_ok()));
+        }
+      });
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_,
+                          std::move(P));
+}
+
+void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query,
+                                      td::Promise<td::BufferSlice> promise) {
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_,
+                          query.offset_, query.max_size_, std::move(promise));
+}
+
 void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query,
                                       td::Promise<td::BufferSlice> promise) {
   auto B = fetch_tl_object<ton_api::Function>(std::move(query), true);
@@ -637,6 +659,15 @@ void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp t
       .release();
 }
 
+void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                                         td::Promise<std::string> promise) {
+  auto &b = choose_neighbour();
+  td::actor::create_actor<DownloadArchiveSlice>(
+      "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), timeout,
+      validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise)))
+      .release();
+}
+
 void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
   CHECK(!handle_);
   handle_ = std::move(handle);
@@ -741,7 +772,7 @@ void FullNodeShardImpl::update_validators(std::vector<PublicKeyHash> public_key_
     authorized_keys.emplace(key, overlay::Overlays::max_fec_broadcast_size());
   }
 
-  rules_ = overlay::OverlayPrivacyRules{overlay::Overlays::max_simple_broadcast_size(), std::move(authorized_keys)};
+  rules_ = overlay::OverlayPrivacyRules{1 << 14, std::move(authorized_keys)};
   td::actor::send_closure(overlays_, &overlay::Overlays::set_privacy_rules, adnl_id_, overlay_id_, rules_);
 
   if (update_cert) {

validator/full-node-shard.h

@@ -55,6 +55,8 @@ class FullNodeShard : public td::actor::Actor {
                                          td::Promise<td::BufferSlice> promise) = 0;
   virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout,
                                    td::Promise<std::vector<BlockIdExt>> promise) = 0;
+  virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                                td::Promise<std::string> promise) = 0;
 
   virtual void set_handle(BlockHandle handle, td::Promise<td::Unit> promise) = 0;
 

validator/full-node-shard.hpp

@@ -61,7 +61,7 @@ class FullNodeShardImpl : public FullNodeShard {
     return 1;
   }
   static constexpr td::uint32 proto_version() {
-    return 1;
+    return 2;
   }
   static constexpr td::uint64 proto_capabilities() {
     return 0;
@@ -120,6 +120,10 @@ class FullNodeShardImpl : public FullNodeShard {
                      td::Promise<td::BufferSlice> promise);
   void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query,
                      td::Promise<td::BufferSlice> promise);
+  void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query,
+                     td::Promise<td::BufferSlice> promise);
+  void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query,
+                     td::Promise<td::BufferSlice> promise);
   // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query,
   //                   td::Promise<td::BufferSlice> promise);
   void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise<td::BufferSlice> promise);
@@ -148,6 +152,8 @@ class FullNodeShardImpl : public FullNodeShard {
                                  td::Promise<td::BufferSlice> promise) override;
   void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout,
                            td::Promise<std::vector<BlockIdExt>> promise) override;
+  void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                        td::Promise<std::string> promise) override;
 
   void set_handle(BlockHandle handle, td::Promise<td::Unit> promise) override;
 

validator/full-node.cpp

@@ -230,6 +230,14 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou
   td::actor::send_closure(shard, &FullNodeShard::get_next_key_blocks, block_id, timeout, std::move(promise));
 }
 
+void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                                    td::Promise<std::string> promise) {
+  auto shard = get_shard(ShardIdFull{masterchainId});
+  CHECK(!shard.empty());
+  td::actor::send_closure(shard, &FullNodeShard::download_archive, masterchain_seqno, std::move(tmp_dir), timeout,
+                          std::move(promise));
+}
+
 td::actor::ActorId<FullNodeShard> FullNodeImpl::get_shard(ShardIdFull shard) {
   while (shards_.count(shard) == 0) {
     if (shard.shard == shardIdAll) {
@@ -392,6 +400,11 @@ void FullNodeImpl::start_up() {
                              td::Promise<std::vector<BlockIdExt>> promise) override {
       td::actor::send_closure(id_, &FullNodeImpl::get_next_key_blocks, block_id, timeout, std::move(promise));
     }
+    void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                          td::Promise<std::string> promise) override {
+      td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout,
+                              std::move(promise));
+    }
 
     void new_key_block(BlockHandle handle) override {
       td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle));

validator/full-node.hpp

@@ -64,6 +64,8 @@ class FullNodeImpl : public FullNode {
   void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout,
                                  td::Promise<td::BufferSlice> promise);
   void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise<std::vector<BlockIdExt>> promise);
+  void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                        td::Promise<std::string> promise);
 
   void got_key_block_proof(td::Ref<ProofLink> proof);
   void got_zero_block_state(td::Ref<ShardState> state);

validator/impl/accept-block.cpp

@@ -41,14 +41,16 @@ using namespace std::literals::string_literals;
 
 AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
                                    td::Ref<ValidatorSet> validator_set, td::Ref<BlockSignatureSet> signatures,
-                                   bool send_broadcast, td::actor::ActorId<ValidatorManager> manager,
-                                   td::Promise<td::Unit> promise)
+                                   td::Ref<BlockSignatureSet> approve_signatures, bool send_broadcast,
+                                   td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise)
     : id_(id)
     , data_(std::move(data))
     , prev_(std::move(prev))
     , validator_set_(std::move(validator_set))
     , signatures_(std::move(signatures))
+    , approve_signatures_(std::move(approve_signatures))
     , is_fake_(false)
+    , is_fork_(false)
     , send_broadcast_(send_broadcast)
     , manager_(manager)
     , promise_(std::move(promise)) {
@@ -66,6 +68,7 @@ AcceptBlockQuery::AcceptBlockQuery(AcceptBlockQuery::IsFake fake, BlockIdExt id,
     , prev_(std::move(prev))
     , validator_set_(std::move(validator_set))
     , is_fake_(true)
+    , is_fork_(false)
     , send_broadcast_(false)
     , manager_(manager)
     , promise_(std::move(promise)) {
@@ -75,6 +78,74 @@ AcceptBlockQuery::AcceptBlockQuery(AcceptBlockQuery::IsFake fake, BlockIdExt id,
   CHECK(prev_.size() > 0);
 }
 
+AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref<BlockData> data,
+                                   td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise)
+    : id_(id)
+    , data_(std::move(data))
+    , is_fake_(true)
+    , is_fork_(true)
+    , send_broadcast_(false)
+    , manager_(manager)
+    , promise_(std::move(promise)) {
+  state_keep_old_hash_.clear();
+  state_old_hash_.clear();
+  state_hash_.clear();
+}
+
+bool AcceptBlockQuery::precheck_header() {
+  VLOG(VALIDATOR_DEBUG) << "precheck_header()";
+  // 0. sanity check
+  CHECK(data_.not_null());
+  block_root_ = data_->root_cell();
+  if (data_->block_id() != id_) {
+    return fatal_error("incorrect block id in block data: "s + data_->block_id().to_str() + " instead of " +
+                       id_.to_str());
+  }
+  // 1. root hash and file hash check
+  RootHash blk_rhash{block_root_->get_hash().bits()};
+  if (blk_rhash != id_.root_hash) {
+    return fatal_error("block root hash mismatch: expected "s + id_.root_hash.to_hex() + ", found " +
+                       blk_rhash.to_hex());
+  }
+  if (is_fake_ || is_fork_) {
+    FileHash blk_fhash;
+    td::sha256(data_->data().as_slice(), blk_fhash.as_slice());
+    if (blk_fhash != id_.file_hash) {
+      return fatal_error("block file hash mismatch: expected "s + id_.file_hash.to_hex() + ", computed " +
+                         blk_fhash.to_hex());
+    }
+  }
+  // 2. check header fields
+  std::vector<ton::BlockIdExt> prev;
+  ton::BlockIdExt mc_blkid;
+  bool after_split;
+  auto res = block::unpack_block_prev_blk_try(block_root_, id_, prev, mc_blkid, after_split);
+  if (res.is_error()) {
+    return fatal_error("invalid block header in AcceptBlock: "s + res.to_string());
+  }
+  if (is_fork_) {
+    prev_ = prev;
+  } else if (prev_ != prev) {
+    return fatal_error("invalid previous block reference(s) in block header");
+  }
+  // 3. unpack header and check vert_seqno fields
+  block::gen::Block::Record blk;
+  block::gen::BlockInfo::Record info;
+  if (!(tlb::unpack_cell(block_root_, blk) && tlb::unpack_cell(blk.info, info))) {
+    return fatal_error("cannot unpack block header");
+  }
+  if (info.vert_seqno_incr && !is_fork_) {
+    return fatal_error("block header has vert_seqno_incr set in an ordinary AcceptBlock");
+  }
+  if (!info.vert_seqno_incr && is_fork_) {
+    return fatal_error("fork block header has no vert_seqno_incr");
+  }
+  if (is_fork_ && !info.key_block) {
+    return fatal_error("fork block is not a key block");
+  }
+  return true;
+}
+
 bool AcceptBlockQuery::create_new_proof() {
   // 0. check block's root hash
   VLOG(VALIDATOR_DEBUG) << "create_new_proof() : start";
@@ -93,7 +164,7 @@ bool AcceptBlockQuery::create_new_proof() {
   block::CurrencyCollection fees;
   ShardIdFull shard;
   if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(blk.info, info) && !info.version &&
-        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && !info.vert_seq_no &&
+        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) &&
         block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) &&
         tlb::unpack_cell(std::move(blk.extra), extra) && block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) &&
         (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) {
@@ -292,6 +363,26 @@ void AcceptBlockQuery::start_up() {
     fatal_error("no real SignatureSet passed to AcceptBlockQuery");
     return;
   }
+  if (!is_fake_ && is_fork_) {
+    fatal_error("a non-fake AcceptBlockQuery for a forced fork block");
+    return;
+  }
+  if (!is_fork_ && !prev_.size()) {
+    fatal_error("no previous blocks passed to AcceptBlockQuery");
+    return;
+  }
+  if (is_fork_ && !is_masterchain()) {
+    fatal_error("cannot accept a non-masterchain fork block");
+    return;
+  }
+  if (is_fork_ && data_.is_null()) {
+    fatal_error("cannot accept a fork block without explicit data");
+    return;
+  }
+  if (data_.not_null() && !precheck_header()) {
+    fatal_error("invalid block header in AcceptBlock");
+    return;
+  }
 
   td::actor::send_closure(
       manager_, &ValidatorManager::get_block_handle, id_, true, [SelfId = actor_id(this)](td::Result<BlockHandle> R) {
@@ -330,8 +421,8 @@ void AcceptBlockQuery::written_block_data() {
   if (is_fake_) {
     signatures_ = Ref<BlockSignatureSetQ>(create_signature_set(std::vector<BlockSignature>{}));
   }
-  td::actor::send_closure(manager_, &ValidatorManager::set_block_signatures, handle_,
-                          signatures_, [SelfId = actor_id(this)](td::Result<td::Unit> R) {
+  td::actor::send_closure(manager_, &ValidatorManager::set_block_signatures, handle_, signatures_,
+                          [SelfId = actor_id(this)](td::Result<td::Unit> R) {
                             check_send_error(SelfId, R) ||
                                 td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::written_block_signatures);
                           });
@@ -365,8 +456,8 @@ void AcceptBlockQuery::written_block_info() {
     td::actor::send_closure(manager_, &ValidatorManager::wait_prev_block_state, handle_, priority(), timeout_,
                             std::move(P));
   } else {
-    td::actor::send_closure(manager_, &ValidatorManager::wait_block_data, handle_, priority(),
-                            timeout_, [SelfId = actor_id(this)](td::Result<td::Ref<BlockData>> R) {
+    td::actor::send_closure(manager_, &ValidatorManager::wait_block_data, handle_, priority(), timeout_,
+                            [SelfId = actor_id(this)](td::Result<td::Ref<BlockData>> R) {
                               check_send_error(SelfId, R) ||
                                   td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_block_data,
                                                                R.move_as_ok());
@@ -382,6 +473,10 @@ void AcceptBlockQuery::got_block_data(td::Ref<BlockData> data) {
     fatal_error("block data does not contain a root cell");
     return;
   }
+  if (!precheck_header()) {
+    fatal_error("invalid block header in AcceptBlock");
+    return;
+  }
   if (handle_->received()) {
     written_block_data();
   } else {
@@ -406,8 +501,8 @@ void AcceptBlockQuery::got_prev_state(td::Ref<ShardState> state) {
 
   handle_->set_split(state_->before_split());
 
-  td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_,
-                          state_, [SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
+  td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, state_,
+                          [SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
                             check_send_error(SelfId, R) ||
                                 td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::written_state, R.move_as_ok());
                           });
@@ -479,8 +574,8 @@ void AcceptBlockQuery::got_last_mc_block(std::pair<td::Ref<MasterchainState>, Bl
   if (last_mc_id_.id.seqno < mc_blkid_.id.seqno) {
     VLOG(VALIDATOR_DEBUG) << "shardchain block refers to newer masterchain block " << mc_blkid_.to_str()
                           << ", trying to obtain it";
-    td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(),
-                                  timeout_, [SelfId = actor_id(this)](td::Result<Ref<ShardState>> R) {
+    td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(), timeout_,
+                                  [SelfId = actor_id(this)](td::Result<Ref<ShardState>> R) {
                                     check_send_error(SelfId, R) ||
                                         td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_mc_state,
                                                                      R.move_as_ok());
@@ -586,7 +681,7 @@ void AcceptBlockQuery::require_proof_link(BlockIdExt id) {
   CHECK(ton::ShardIdFull(id) == ton::ShardIdFull(id_));
   CHECK(id.id.seqno == id_.id.seqno - 1 - proof_links_.size());
   td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_proof_link_short, id, timeout_,
-                                [ SelfId = actor_id(this), id ](td::Result<Ref<ProofLink>> R) {
+                                [SelfId = actor_id(this), id](td::Result<Ref<ProofLink>> R) {
                                   check_send_error(SelfId, R) ||
                                       td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_proof_link, id,
                                                                    R.move_as_ok());
@@ -785,7 +880,7 @@ void AcceptBlockQuery::written_block_info_2() {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
       check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::applied);
     });
-    run_apply_block_query(handle_->id(), data_, manager_, timeout_, std::move(P));
+    run_apply_block_query(handle_->id(), data_, handle_->id(), manager_, timeout_, std::move(P));
   } else {
     applied();
   }

validator/impl/accept-block.hpp

@@ -47,12 +47,16 @@ using td::Ref;
 class AcceptBlockQuery : public td::actor::Actor {
  public:
   struct IsFake {};
+  struct ForceFork {};
   AcceptBlockQuery(BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
-                   td::Ref<ValidatorSet> validator_set, td::Ref<BlockSignatureSet> signatures, bool send_broadcast,
+                   td::Ref<ValidatorSet> validator_set, td::Ref<BlockSignatureSet> signatures,
+                   td::Ref<BlockSignatureSet> approve_signatures, bool send_broadcast,
                    td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise);
   AcceptBlockQuery(IsFake fake, BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
                    td::Ref<ValidatorSet> validator_set, td::actor::ActorId<ValidatorManager> manager,
                    td::Promise<td::Unit> promise);
+  AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref<BlockData> data,
+                   td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise);
 
  private:
   static constexpr td::uint32 priority() {
@@ -90,7 +94,9 @@ class AcceptBlockQuery : public td::actor::Actor {
   std::vector<BlockIdExt> prev_;
   Ref<ValidatorSetQ> validator_set_;
   Ref<BlockSignatureSetQ> signatures_;
+  Ref<BlockSignatureSetQ> approve_signatures_;
   bool is_fake_;
+  bool is_fork_;
   bool send_broadcast_;
   bool ancestors_split_{false}, is_key_block_{false};
   td::Timestamp timeout_ = td::Timestamp::in(600.0);
@@ -128,6 +134,7 @@ class AcceptBlockQuery : public td::actor::Actor {
   static bool check_send_error(td::actor::ActorId<AcceptBlockQuery> SelfId, td::Result<T>& res) {
     return res.is_error() && check_send_error(std::move(SelfId), res.move_as_error());
   }
+  bool precheck_header();
   bool create_new_proof();
   bool unpack_proof_link(BlockIdExt id, Ref<ProofLink> proof);
 

validator/impl/check-proof.cpp

@@ -165,7 +165,7 @@ bool CheckProof::init_parse(bool is_aux) {
   block::gen::ExtBlkRef::Record mcref;  // _ ExtBlkRef = BlkMasterInfo;
   ShardIdFull shard;
   if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(blk.info, info) && !info.version &&
-        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && !info.vert_seq_no &&
+        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) &&
         block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) &&
         block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) &&
         (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) {

validator/impl/fabric.cpp

@@ -119,7 +119,8 @@ void run_accept_block_query(BlockIdExt id, td::Ref<BlockData> data, std::vector<
                             td::Ref<BlockSignatureSet> approve_signatures, bool send_broadcast,
                             td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise) {
   td::actor::create_actor<AcceptBlockQuery>("accept", id, std::move(data), prev, std::move(validator_set),
-                                            std::move(signatures), send_broadcast, manager, std::move(promise))
+                                            std::move(signatures), std::move(approve_signatures), send_broadcast,
+                                            manager, std::move(promise))
       .release();
 }
 
@@ -134,13 +135,16 @@ void run_fake_accept_block_query(BlockIdExt id, td::Ref<BlockData> data, std::ve
 
 void run_hardfork_accept_block_query(BlockIdExt id, td::Ref<BlockData> data,
                                      td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise) {
-  promise.set_error(td::Status::Error(ErrorCode::error, "not implemented"));
+  td::actor::create_actor<AcceptBlockQuery>("fork/accept", AcceptBlockQuery::ForceFork(), id, std::move(data),
+                                            std::move(manager), std::move(promise))
+      .release();
 }
 
-void run_apply_block_query(BlockIdExt id, td::Ref<BlockData> block, td::actor::ActorId<ValidatorManager> manager,
-                           td::Timestamp timeout, td::Promise<td::Unit> promise) {
-  td::actor::create_actor<ApplyBlock>(PSTRING() << "apply " << id, id, std::move(block), manager, timeout,
-                                      std::move(promise))
+void run_apply_block_query(BlockIdExt id, td::Ref<BlockData> block, BlockIdExt masterchain_block_id,
+                           td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
+                           td::Promise<td::Unit> promise) {
+  td::actor::create_actor<ApplyBlock>(PSTRING() << "apply " << id, id, std::move(block), masterchain_block_id, manager,
+                                      timeout, std::move(promise))
       .release();
 }
 

validator/impl/liteserver.cpp

@@ -192,7 +192,7 @@ void LiteQuery::perform_getMasterchainInfo(int mode) {
   }
   td::actor::send_closure_later(
       manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block,
-      [ Self = actor_id(this), mode ](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res) {
+      [Self = actor_id(this), mode](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
         } else {
@@ -230,7 +230,7 @@ void LiteQuery::perform_getBlock(BlockIdExt blkid) {
     return;
   }
   td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
-                                [ Self = actor_id(this), blkid ](td::Result<Ref<ton::validator::BlockData>> res) {
+                                [Self = actor_id(this), blkid](td::Result<Ref<ton::validator::BlockData>> res) {
                                   if (res.is_error()) {
                                     td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                   } else {
@@ -256,7 +256,7 @@ void LiteQuery::perform_getBlockHeader(BlockIdExt blkid, int mode) {
     return;
   }
   td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
-                                [ Self = actor_id(this), blkid, mode ](td::Result<Ref<ton::validator::BlockData>> res) {
+                                [Self = actor_id(this), blkid, mode](td::Result<Ref<ton::validator::BlockData>> res) {
                                   if (res.is_error()) {
                                     td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                   } else {
@@ -371,7 +371,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) {
   }
   if (blkid.id.seqno) {
     td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
-                                  [ Self = actor_id(this), blkid ](td::Result<Ref<ton::validator::ShardState>> res) {
+                                  [Self = actor_id(this), blkid](td::Result<Ref<ton::validator::ShardState>> res) {
                                     if (res.is_error()) {
                                       td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                     } else {
@@ -381,7 +381,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) {
                                   });
   } else {
     td::actor::send_closure_later(manager_, &ValidatorManager::get_zero_state, blkid,
-                                  [ Self = actor_id(this), blkid ](td::Result<td::BufferSlice> res) {
+                                  [Self = actor_id(this), blkid](td::Result<td::BufferSlice> res) {
                                     if (res.is_error()) {
                                       td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                     } else {
@@ -440,7 +440,7 @@ bool LiteQuery::request_mc_block_data(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<Ref<BlockData>> res) {
+      [Self = actor_id(this), blkid](td::Result<Ref<BlockData>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : "));
@@ -466,7 +466,7 @@ bool LiteQuery::request_mc_proof(BlockIdExt blkid, int mode) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_block_proof_from_db_short, blkid,
-      [ Self = actor_id(this), blkid, mode ](td::Result<Ref<Proof>> res) {
+      [Self = actor_id(this), blkid, mode](td::Result<Ref<Proof>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load proof for "s + blkid.to_str() + " : "));
@@ -488,7 +488,7 @@ bool LiteQuery::request_mc_block_state(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<Ref<ShardState>> res) {
+      [Self = actor_id(this), blkid](td::Result<Ref<ShardState>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : "));
@@ -519,7 +519,7 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<Ref<ShardState>> res) {
+      [Self = actor_id(this), blkid](td::Result<Ref<ShardState>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : "));
@@ -541,7 +541,7 @@ bool LiteQuery::request_block_data(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<Ref<BlockData>> res) {
+      [Self = actor_id(this), blkid](td::Result<Ref<BlockData>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : "));
@@ -563,7 +563,7 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_block_proof_link_from_db_short, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<Ref<ProofLink>> res) {
+      [Self = actor_id(this), blkid](td::Result<Ref<ProofLink>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : "));
@@ -588,7 +588,7 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_zero_state, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<td::BufferSlice> res) {
+      [Self = actor_id(this), blkid](td::Result<td::BufferSlice> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : "));
@@ -632,7 +632,7 @@ void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain,
     LOG(INFO) << "sending a get_top_masterchain_state_block query to manager";
     td::actor::send_closure_later(
         manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block,
-        [Self = actor_id(this)](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res)->void {
+        [Self = actor_id(this)](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res) -> void {
           if (res.is_error()) {
             td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
           } else {
@@ -1067,14 +1067,14 @@ void LiteQuery::continue_getTransactions(unsigned remaining, bool exact) {
              << " " << trans_lt_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_block_by_lt_from_db, ton::extract_addr_prefix(acc_workchain_, acc_addr_),
-      trans_lt_, [ Self = actor_id(this), remaining, manager = manager_ ](td::Result<BlockIdExt> res) {
+      trans_lt_, [Self = actor_id(this), remaining, manager = manager_](td::Result<BlockHandle> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, res.move_as_error(), ton::BlockIdExt{});
         } else {
-          auto blkid = res.move_as_ok();
-          LOG(DEBUG) << "requesting data for block " << blkid.to_str();
-          td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db_short, blkid,
-                                        [Self, blkid, remaining](td::Result<Ref<BlockData>> res) {
+          auto handle = res.move_as_ok();
+          LOG(DEBUG) << "requesting data for block " << handle->id().to_str();
+          td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle,
+                                        [Self, blkid = handle->id(), remaining](td::Result<Ref<BlockData>> res) {
                                           if (res.is_error()) {
                                             td::actor::send_closure(Self, &LiteQuery::abort_getTransactions,
                                                                     res.move_as_error(), blkid);
@@ -1141,7 +1141,7 @@ void LiteQuery::perform_getShardInfo(BlockIdExt blkid, ShardIdFull shard, bool e
 void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector<int> param_list) {
   LOG(INFO) << "started a getConfigParams(" << blkid.to_str() << ", " << mode << ", <list of " << param_list.size()
             << " parameters>) liteserver query";
-  set_continuation([ this, mode, param_list = std::move(param_list) ]() mutable {
+  set_continuation([this, mode, param_list = std::move(param_list)]() mutable {
     continue_getConfigParams(mode, std::move(param_list));
   });
   request_mc_block_data_state(blkid);
@@ -1294,14 +1294,14 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni
   LOG(INFO) << "performing a lookupBlock(" << blkid.to_str() << ", " << mode << ", " << lt << ", " << utime
             << ") query";
   auto P = td::PromiseCreator::lambda(
-      [ Self = actor_id(this), manager = manager_, mode = (mode >> 4) ](td::Result<BlockIdExt> res) {
+      [Self = actor_id(this), manager = manager_, mode = (mode >> 4)](td::Result<BlockHandle> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
         } else {
-          auto blkid = res.move_as_ok();
-          LOG(DEBUG) << "requesting data for block " << blkid.to_str();
-          td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db_short, blkid,
-                                        [Self, blkid, mode](td::Result<Ref<BlockData>> res) {
+          auto handle = res.move_as_ok();
+          LOG(DEBUG) << "requesting data for block " << handle->id().to_str();
+          td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle,
+                                        [Self, blkid = handle->id(), mode](td::Result<Ref<BlockData>> res) {
                                           if (res.is_error()) {
                                             td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                           } else {
@@ -1449,7 +1449,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to,
     if (mode & 0x1000) {
       BlockIdExt bblk = (from.seqno() > to.seqno()) ? from : to;
       td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, bblk,
-                                    [ Self = actor_id(this), from, to, bblk, mode ](td::Result<Ref<ShardState>> res) {
+                                    [Self = actor_id(this), from, to, bblk, mode](td::Result<Ref<ShardState>> res) {
                                       if (res.is_error()) {
                                         td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                       } else {
@@ -1461,7 +1461,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to,
     } else {
       td::actor::send_closure_later(
           manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block,
-          [ Self = actor_id(this), from, to, mode ](td::Result<std::pair<Ref<MasterchainState>, BlockIdExt>> res) {
+          [Self = actor_id(this), from, to, mode](td::Result<std::pair<Ref<MasterchainState>, BlockIdExt>> res) {
             if (res.is_error()) {
               td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
             } else {
@@ -1474,7 +1474,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to,
   } else if (mode & 2) {
     td::actor::send_closure_later(
         manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block,
-        [ Self = actor_id(this), from, mode ](td::Result<std::pair<Ref<MasterchainState>, BlockIdExt>> res) {
+        [Self = actor_id(this), from, mode](td::Result<std::pair<Ref<MasterchainState>, BlockIdExt>> res) {
           if (res.is_error()) {
             td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
           } else {
@@ -1485,7 +1485,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to,
         });
   } else {
     td::actor::send_closure_later(manager_, &ton::validator::ValidatorManager::get_shard_client_state, false,
-                                  [ Self = actor_id(this), from, mode ](td::Result<BlockIdExt> res) {
+                                  [Self = actor_id(this), from, mode](td::Result<BlockIdExt> res) {
                                     if (res.is_error()) {
                                       td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                     } else {

validator/impl/proof.cpp

@@ -88,6 +88,7 @@ td::Result<ProofLink::BasicHeaderInfo> ProofLinkQ::get_basic_header_info() const
     }
     res.cc_seqno = info.gen_catchain_seqno;
     res.utime = info.gen_utime;
+    res.end_lt = info.end_lt;
     res.validator_set_hash = info.gen_validator_list_hash_short;
     res.prev_key_mc_seqno = info.prev_key_block_seqno;
     return res;

validator/impl/validate-query.cpp

@@ -430,6 +430,7 @@ bool ValidateQuery::init_parse() {
     return reject_query("a non-masterchain block cannot be a key block");
   }
   if (info.vert_seqno_incr) {
+    // what about non-masterchain blocks?
     return reject_query("new blocks cannot have vert_seqno_incr set");
   }
   if (info.after_merge != after_merge_) {
@@ -1620,12 +1621,13 @@ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block:
                                   << " has unchanged catchain seqno " << cc_seqno
                                   << ", but it must have been updated for all shards");
   }
-  if (!cc_updated && !info.before_merge_ && old_before_merge && !workchain_created) {
+  bool bm_cleared = !info.before_merge_ && old_before_merge;
+  if (!cc_updated && bm_cleared && !workchain_created) {
     return reject_query(PSTRING() << "new shard configuration for shard " << shard.to_str()
                                   << " has unchanged catchain seqno " << cc_seqno
                                   << " while the before_merge bit has been cleared");
   }
-  if (cc_updated && (!update_shard_cc_ || (!info.before_merge_ && old_before_merge))) {
+  if (cc_updated && !(update_shard_cc_ || bm_cleared)) {
     return reject_query(PSTRING() << "new shard configuration for shard " << shard.to_str()
                                   << " has increased catchain seqno " << cc_seqno << " without a good reason");
   }

validator/import-db-slice.cpp

@@ -0,0 +1,354 @@
+#include "import-db-slice.hpp"
+#include "validator/db/fileref.hpp"
+#include "td/utils/overloaded.h"
+#include "validator/fabric.h"
+#include "td/actor/MultiPromise.h"
+#include "common/checksum.h"
+#include "td/utils/port/path.h"
+
+namespace ton {
+
+namespace validator {
+
+ArchiveImporter::ArchiveImporter(std::string path, td::Ref<MasterchainState> state, BlockSeqno shard_client_seqno,
+                                 td::Ref<ValidatorManagerOptions> opts, td::actor::ActorId<ValidatorManager> manager,
+                                 td::Promise<std::vector<BlockSeqno>> promise)
+    : path_(std::move(path))
+    , state_(std::move(state))
+    , shard_client_seqno_(shard_client_seqno)
+    , opts_(std::move(opts))
+    , manager_(manager)
+    , promise_(std::move(promise)) {
+}
+
+void ArchiveImporter::start_up() {
+  auto R = Package::open(path_, false, false);
+  if (R.is_error()) {
+    abort_query(R.move_as_error());
+    return;
+  }
+  package_ = std::make_shared<Package>(R.move_as_ok());
+
+  bool fail = false;
+  package_->iterate([&](std::string filename, td::BufferSlice data, td::uint64 offset) -> bool {
+    auto F = FileReference::create(filename);
+    if (F.is_error()) {
+      abort_query(F.move_as_error());
+      fail = true;
+      return false;
+    }
+    auto f = F.move_as_ok();
+
+    BlockIdExt b;
+    bool is_proof = false;
+    bool ignore = true;
+
+    f.ref().visit(td::overloaded(
+        [&](const fileref::Proof &p) {
+          b = p.block_id;
+          ignore = !b.is_masterchain();
+          is_proof = true;
+        },
+        [&](const fileref::ProofLink &p) {
+          b = p.block_id;
+          ignore = b.is_masterchain();
+          is_proof = true;
+        },
+        [&](const fileref::Block &p) {
+          b = p.block_id;
+          ignore = false;
+          is_proof = false;
+        },
+        [&](const auto &p) { ignore = true; }));
+
+    if (!ignore) {
+      blocks_[b][is_proof ? 0 : 1] = offset;
+      if (b.is_masterchain()) {
+        masterchain_blocks_[b.seqno()] = b;
+      }
+    }
+    return true;
+  });
+
+  if (fail) {
+    return;
+  }
+
+  if (masterchain_blocks_.size() == 0) {
+    abort_query(td::Status::Error(ErrorCode::notready, "archive does not contain any masterchain blocks"));
+    return;
+  }
+
+  auto seqno = masterchain_blocks_.begin()->first;
+
+  check_masterchain_block(seqno);
+}
+
+void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) {
+  auto it = masterchain_blocks_.find(seqno);
+  if (it == masterchain_blocks_.end()) {
+    if (seqno == 0) {
+      abort_query(td::Status::Error(ErrorCode::notready, "no new blocks"));
+      return;
+    }
+    checked_all_masterchain_blocks(seqno - 1);
+    return;
+  }
+  if (seqno < state_->get_block_id().seqno()) {
+    if (!state_->check_old_mc_block_id(it->second)) {
+      abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id"));
+      return;
+    }
+    check_masterchain_block(seqno + 1);
+  } else if (seqno == state_->get_block_id().seqno()) {
+    if (state_->get_block_id() != it->second) {
+      abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id"));
+      return;
+    }
+    check_masterchain_block(seqno + 1);
+  } else {
+    if (seqno != state_->get_block_id().seqno() + 1) {
+      abort_query(td::Status::Error(ErrorCode::protoviolation, "hole in masterchain seqno"));
+      return;
+    }
+    auto it2 = blocks_.find(it->second);
+    CHECK(it2 != blocks_.end());
+
+    auto R1 = package_->read(it2->second[0]);
+    if (R1.is_error()) {
+      abort_query(R1.move_as_error());
+      return;
+    }
+
+    auto proofR = create_proof(it->second, std::move(R1.move_as_ok().second));
+    if (proofR.is_error()) {
+      abort_query(proofR.move_as_error());
+      return;
+    }
+
+    auto R2 = package_->read(it2->second[1]);
+    if (R2.is_error()) {
+      abort_query(R2.move_as_error());
+      return;
+    }
+
+    if (sha256_bits256(R2.ok().second.as_slice()) != it->second.file_hash) {
+      abort_query(td::Status::Error(ErrorCode::protoviolation, "bad block file hash"));
+      return;
+    }
+    auto dataR = create_block(it->second, std::move(R2.move_as_ok().second));
+    if (dataR.is_error()) {
+      abort_query(dataR.move_as_error());
+      return;
+    }
+
+    auto proof = proofR.move_as_ok();
+    auto data = dataR.move_as_ok();
+
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = state_->get_block_id(),
+                                         data](td::Result<BlockHandle> R) mutable {
+      if (R.is_error()) {
+        td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error());
+        return;
+      }
+      auto handle = R.move_as_ok();
+      CHECK(!handle->merge_before());
+      if (handle->one_prev(true) != id) {
+        td::actor::send_closure(SelfId, &ArchiveImporter::abort_query,
+                                td::Status::Error(ErrorCode::protoviolation, "prev block mismatch"));
+        return;
+      }
+      td::actor::send_closure(SelfId, &ArchiveImporter::checked_masterchain_proof, std::move(handle), std::move(data));
+    });
+
+    run_check_proof_query(it->second, std::move(proof), manager_, td::Timestamp::in(2.0), std::move(P), state_,
+                          opts_->is_hardfork(it->second));
+  }
+}
+
+void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Ref<BlockData> data) {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Unit> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveImporter::applied_masterchain_block, std::move(handle));
+  });
+  run_apply_block_query(handle->id(), std::move(data), handle->id(), manager_, td::Timestamp::in(10.0), std::move(P));
+}
+
+void ArchiveImporter::applied_masterchain_block(BlockHandle handle) {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveImporter::got_new_materchain_state,
+                            td::Ref<MasterchainState>(R.move_as_ok()));
+  });
+  td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P));
+}
+
+void ArchiveImporter::got_new_materchain_state(td::Ref<MasterchainState> state) {
+  state_ = std::move(state);
+  check_masterchain_block(state_->get_block_id().seqno() + 1);
+}
+
+void ArchiveImporter::checked_all_masterchain_blocks(BlockSeqno seqno) {
+  max_shard_client_seqno_ = seqno;
+  check_next_shard_client_seqno(shard_client_seqno_ + 1);
+}
+
+void ArchiveImporter::check_next_shard_client_seqno(BlockSeqno seqno) {
+  if (seqno > max_shard_client_seqno_) {
+    finish_query();
+    return;
+  }
+
+  if (seqno == max_shard_client_seqno_) {
+    got_masterchain_state(state_);
+  } else {
+    BlockIdExt b;
+    bool f = state_->get_old_mc_block_id(seqno, b);
+    CHECK(f);
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
+      R.ensure();
+      td::actor::send_closure(SelfId, &ArchiveImporter::got_masterchain_state,
+                              td::Ref<MasterchainState>{R.move_as_ok()});
+    });
+    td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, b, std::move(P));
+  }
+}
+
+void ArchiveImporter::got_masterchain_state(td::Ref<MasterchainState> state) {
+  auto s = state->get_shards();
+
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), seqno = state->get_block_id().seqno()](td::Result<td::Unit> R) {
+        if (R.is_error()) {
+          td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error());
+        } else {
+          td::actor::send_closure(SelfId, &ArchiveImporter::check_next_shard_client_seqno, seqno + 1);
+        }
+      });
+
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise(std::move(P));
+
+  for (auto &shard : s) {
+    apply_shard_block(shard->top_block_id(), state->get_block_id(), ig.get_promise());
+  }
+}
+
+void ArchiveImporter::apply_shard_block(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                        td::Promise<td::Unit> promise) {
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), masterchain_block_id, promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+        R.ensure();
+        td::actor::send_closure(SelfId, &ArchiveImporter::apply_shard_block_cont1, R.move_as_ok(), masterchain_block_id,
+                                std::move(promise));
+      });
+  td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, true, std::move(P));
+}
+
+void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt masterchain_block_id,
+                                              td::Promise<td::Unit> promise) {
+  if (handle->is_applied()) {
+    promise.set_value(td::Unit());
+    return;
+  }
+
+  auto it = blocks_.find(handle->id());
+  if (it == blocks_.end()) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "no proof for shard block"));
+    return;
+  }
+  TRY_RESULT_PROMISE(promise, data, package_->read(it->second[0]));
+  TRY_RESULT_PROMISE(promise, proof, create_proof_link(handle->id(), std::move(data.second)));
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle, masterchain_block_id,
+                                       promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+    if (R.is_error()) {
+      promise.set_error(R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveImporter::apply_shard_block_cont2, std::move(handle),
+                              masterchain_block_id, std::move(promise));
+    }
+  });
+  run_check_proof_link_query(handle->id(), std::move(proof), manager_, td::Timestamp::in(10.0), std::move(P));
+}
+
+void ArchiveImporter::apply_shard_block_cont2(BlockHandle handle, BlockIdExt masterchain_block_id,
+                                              td::Promise<td::Unit> promise) {
+  if (handle->is_applied()) {
+    promise.set_value(td::Unit());
+    return;
+  }
+  CHECK(handle->id().seqno() > 0);
+
+  if (!handle->merge_before() && handle->one_prev(true).shard_full() == handle->id().shard_full()) {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle, masterchain_block_id,
+                                         promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+      if (R.is_error()) {
+        promise.set_error(R.move_as_error());
+      } else {
+        td::actor::send_closure(SelfId, &ArchiveImporter::apply_shard_block_cont3, std::move(handle),
+                                masterchain_block_id, std::move(promise));
+      }
+    });
+    apply_shard_block(handle->one_prev(true), masterchain_block_id, std::move(P));
+  } else {
+    td::MultiPromise mp;
+    auto ig = mp.init_guard();
+    ig.add_promise(std::move(promise));
+    check_shard_block_applied(handle->one_prev(true), ig.get_promise());
+    if (handle->merge_before()) {
+      check_shard_block_applied(handle->one_prev(false), ig.get_promise());
+    }
+  }
+}
+
+void ArchiveImporter::apply_shard_block_cont3(BlockHandle handle, BlockIdExt masterchain_block_id,
+                                              td::Promise<td::Unit> promise) {
+  auto it = blocks_.find(handle->id());
+  CHECK(it != blocks_.end());
+  TRY_RESULT_PROMISE(promise, data, package_->read(it->second[1]));
+  if (sha256_bits256(data.second.as_slice()) != handle->id().file_hash) {
+    promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad block file hash"));
+    return;
+  }
+  TRY_RESULT_PROMISE(promise, block, create_block(handle->id(), std::move(data.second)));
+
+  run_apply_block_query(handle->id(), std::move(block), masterchain_block_id, manager_, td::Timestamp::in(10.0),
+                        std::move(promise));
+}
+
+void ArchiveImporter::check_shard_block_applied(BlockIdExt block_id, td::Promise<td::Unit> promise) {
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          auto handle = R.move_as_ok();
+          if (!handle->is_applied()) {
+            promise.set_error(td::Status::Error(ErrorCode::notready, "not applied"));
+          } else {
+            promise.set_value(td::Unit());
+          }
+        }
+      });
+  td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, false, std::move(P));
+}
+
+void ArchiveImporter::abort_query(td::Status error) {
+  if (promise_) {
+    promise_.set_error(std::move(error));
+    td::unlink(path_).ensure();
+  }
+  stop();
+}
+void ArchiveImporter::finish_query() {
+  if (promise_) {
+    promise_.set_value(std::vector<BlockSeqno>(state_->get_block_id().seqno(), max_shard_client_seqno_));
+    td::unlink(path_).ensure();
+  }
+  stop();
+}
+
+}  // namespace validator
+
+}  // namespace ton

validator/import-db-slice.hpp

@@ -0,0 +1,54 @@
+#pragma once
+
+#include "td/actor/actor.h"
+#include "validator/interfaces/validator-manager.h"
+#include "validator/db/package.hpp"
+
+namespace ton {
+
+namespace validator {
+
+class ArchiveImporter : public td::actor::Actor {
+ public:
+  ArchiveImporter(std::string path, td::Ref<MasterchainState> state, BlockSeqno shard_client_seqno,
+                  td::Ref<ValidatorManagerOptions> opts, td::actor::ActorId<ValidatorManager> manager,
+                  td::Promise<std::vector<BlockSeqno>> promise);
+  void start_up() override;
+
+  void abort_query(td::Status error);
+  void finish_query();
+
+  void check_masterchain_block(BlockSeqno seqno);
+  void checked_masterchain_proof(BlockHandle handle, td::Ref<BlockData> data);
+  void applied_masterchain_block(BlockHandle handle);
+  void got_new_materchain_state(td::Ref<MasterchainState> state);
+  void checked_all_masterchain_blocks(BlockSeqno seqno);
+
+  void check_next_shard_client_seqno(BlockSeqno seqno);
+  void got_masterchain_state(td::Ref<MasterchainState> state);
+  void apply_shard_block(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise);
+  void apply_shard_block_cont1(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise);
+  void apply_shard_block_cont2(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise);
+  void apply_shard_block_cont3(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise);
+  void check_shard_block_applied(BlockIdExt block_id, td::Promise<td::Unit> promise);
+
+ private:
+  std::string path_;
+  td::Ref<MasterchainState> state_;
+  BlockSeqno shard_client_seqno_;
+  BlockSeqno max_shard_client_seqno_;
+
+  td::Ref<ValidatorManagerOptions> opts_;
+
+  std::shared_ptr<Package> package_;
+
+  td::actor::ActorId<ValidatorManager> manager_;
+  td::Promise<std::vector<BlockSeqno>> promise_;
+
+  std::map<BlockSeqno, BlockIdExt> masterchain_blocks_;
+  std::map<BlockIdExt, std::array<td::uint64, 2>> blocks_;
+};
+
+}  // namespace validator
+
+}  // namespace ton

validator/interfaces/block-handle.h

@@ -32,8 +32,8 @@ struct BlockHandleInterface {
  public:
   virtual BlockIdExt id() const = 0;
   virtual bool received() const = 0;
-  virtual bool moved_to_storage() const = 0;
   virtual bool moved_to_archive() const = 0;
+  virtual bool handle_moved_to_archive() const = 0;
   virtual bool deleted() const = 0;
   virtual bool inited_next_left() const = 0;
   virtual bool inited_next_right() const = 0;
@@ -49,6 +49,7 @@ struct BlockHandleInterface {
   virtual bool inited_split_after() const = 0;
   virtual bool inited_merge_before() const = 0;
   virtual bool inited_is_key_block() const = 0;
+  virtual bool inited_masterchain_ref_block() const = 0;
   virtual bool split_after() const = 0;
   virtual bool merge_before() const = 0;
   virtual bool is_key_block() const = 0;
@@ -60,6 +61,7 @@ struct BlockHandleInterface {
   virtual bool is_zero() const = 0;
   virtual bool is_archived() const = 0;
   virtual bool is_applied() const = 0;
+  virtual BlockSeqno masterchain_ref_block() const = 0;
   virtual std::vector<BlockIdExt> prev() const = 0;
   virtual BlockIdExt one_prev(bool left) const = 0;
   virtual std::vector<BlockIdExt> next() const = 0;
@@ -83,8 +85,8 @@ struct BlockHandleInterface {
   virtual void set_next(BlockIdExt next) = 0;
   virtual void set_prev(BlockIdExt prev) = 0;
   virtual void set_received() = 0;
-  virtual void set_moved_to_storage() = 0;
   virtual void set_moved_to_archive() = 0;
+  virtual void set_handle_moved_to_archive() = 0;
   virtual void set_deleted() = 0;
   virtual void set_split(bool value) = 0;
   virtual void set_merge(bool value) = 0;
@@ -94,6 +96,7 @@ struct BlockHandleInterface {
   virtual void set_deleted_state_boc() = 0;
   virtual void set_archived() = 0;
   virtual void set_applied() = 0;
+  virtual void set_masterchain_ref_block(BlockSeqno seqno) = 0;
 
   virtual void unsafe_clear_applied() = 0;
   virtual void unsafe_clear_next() = 0;

validator/interfaces/db.h

@@ -72,9 +72,9 @@ class Db : public td::actor::Actor {
   virtual void get_block_handle(BlockIdExt id, td::Promise<BlockHandle> promise) = 0;
 
   virtual void apply_block(BlockHandle handle, td::Promise<td::Unit> promise) = 0;
-  virtual void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockIdExt> promise) = 0;
-  virtual void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockIdExt> promise) = 0;
-  virtual void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockIdExt> promise) = 0;
+  virtual void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockHandle> promise) = 0;
+  virtual void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockHandle> promise) = 0;
+  virtual void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockHandle> promise) = 0;
 
   virtual void update_init_masterchain_block(BlockIdExt block, td::Promise<td::Unit> promise) = 0;
   virtual void get_init_masterchain_block(td::Promise<BlockIdExt> promise) = 0;
@@ -95,11 +95,25 @@ class Db : public td::actor::Actor {
   virtual void update_hardforks(std::vector<BlockIdExt> blocks, td::Promise<td::Unit> promise) = 0;
   virtual void get_hardforks(td::Promise<std::vector<BlockIdExt>> promise) = 0;
 
-  virtual void archive(BlockIdExt block_id, td::Promise<td::Unit> promise) = 0;
+  virtual void archive(BlockHandle handle, td::Promise<td::Unit> promise) = 0;
 
   virtual void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) = 0;
 
   virtual void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) = 0;
+
+  virtual void add_key_block_proof(td::Ref<Proof> proof, td::Promise<td::Unit> promise) = 0;
+  virtual void add_key_block_proof_link(td::Ref<ProofLink> proof_link, td::Promise<td::Unit> promise) = 0;
+  virtual void get_key_block_proof(BlockIdExt block_id, td::Promise<td::Ref<Proof>> promise) = 0;
+  virtual void get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::Ref<ProofLink>> promise) = 0;
+  virtual void check_key_block_proof_exists(BlockIdExt block_id, td::Promise<bool> promise) = 0;
+  virtual void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise<bool> promise) = 0;
+
+  virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) = 0;
+  virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                                 td::Promise<td::BufferSlice> promise) = 0;
+  virtual void set_async_mode(bool mode, td::Promise<td::Unit> promise) = 0;
+
+  virtual void run_gc(UnixTime ts) = 0;
 };
 
 }  // namespace validator

validator/interfaces/proof.h

@@ -29,6 +29,7 @@ class ProofLink : public td::CntObject {
  public:
   struct BasicHeaderInfo {
     UnixTime utime;
+    LogicalTime end_lt;
     CatchainSeqno cc_seqno;
     td::uint32 validator_set_hash;
     BlockSeqno prev_key_mc_seqno;

validator/interfaces/validator-manager.h

@@ -147,6 +147,8 @@ class ValidatorManager : public ValidatorManagerInterface {
   virtual void allow_block_candidate_gc(BlockIdExt block_id, td::Promise<bool> promise) = 0;
   virtual void allow_block_info_gc(BlockIdExt block_id, td::Promise<bool> promise) = 0;
 
+  virtual void archive(BlockHandle handle, td::Promise<td::Unit> promise) = 0;
+
   virtual void check_is_hardfork(BlockIdExt block_id, td::Promise<bool> promise) = 0;
   virtual void get_vertical_seqno(BlockSeqno seqno, td::Promise<td::uint32> promise) = 0;
 

validator/invariants.hpp

@@ -45,7 +45,7 @@ class ValidatorInvariants {
     CHECK(handle->inited_merge_before());
     CHECK(handle->inited_split_after());
     CHECK(handle->inited_prev());
-    CHECK(handle->inited_signatures());
+    CHECK(handle->inited_signatures() || handle->is_applied());
     CHECK(handle->inited_state_root_hash());
     CHECK(handle->inited_logical_time());
     CHECK(handle->inited_unix_time());

validator/manager-disk.cpp

@@ -223,6 +223,42 @@ void ValidatorManagerImpl::get_block_proof(BlockHandle handle, td::Promise<td::B
   td::actor::send_closure(db_, &Db::get_block_proof, handle, std::move(P));
 }
 
+void ValidatorManagerImpl::get_key_block_proof(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Ref<Proof>> R) mutable {
+    if (R.is_error()) {
+      promise.set_error(R.move_as_error());
+    } else {
+      auto B = R.move_as_ok();
+      promise.set_value(B->data());
+    }
+  });
+
+  td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P));
+}
+
+void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {
+  auto P = td::PromiseCreator::lambda(
+      [promise = std::move(promise), block_id, db = db_.get()](td::Result<td::Ref<Proof>> R) mutable {
+        if (R.is_error()) {
+          auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Ref<Proof>> R) mutable {
+            if (R.is_error()) {
+              promise.set_error(R.move_as_error());
+            } else {
+              auto B = R.move_as_ok();
+              promise.set_value(B->data());
+            }
+          });
+
+          td::actor::send_closure(db, &Db::get_key_block_proof, block_id, std::move(P));
+        } else {
+          auto B = R.move_as_ok()->export_as_proof_link().move_as_ok();
+          promise.set_value(B->data());
+        }
+      });
+
+  td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P));
+}
+
 void ValidatorManagerImpl::new_external_message(td::BufferSlice data) {
   auto R = create_ext_message(std::move(data));
   if (R.is_ok()) {
@@ -582,17 +618,17 @@ void ValidatorManagerImpl::get_block_proof_link_from_db_short(BlockIdExt block_i
 }
 
 void ValidatorManagerImpl::get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt,
-                                                   td::Promise<BlockIdExt> promise) {
+                                                   td::Promise<BlockHandle> promise) {
   td::actor::send_closure(db_, &Db::get_block_by_lt, account, lt, std::move(promise));
 }
 
 void ValidatorManagerImpl::get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts,
-                                                          td::Promise<BlockIdExt> promise) {
+                                                          td::Promise<BlockHandle> promise) {
   td::actor::send_closure(db_, &Db::get_block_by_unix_time, account, ts, std::move(promise));
 }
 
 void ValidatorManagerImpl::get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno,
-                                                      td::Promise<BlockIdExt> promise) {
+                                                      td::Promise<BlockHandle> promise) {
   td::actor::send_closure(db_, &Db::get_block_by_seqno, account, seqno, std::move(promise));
 }
 

validator/manager-disk.hpp

@@ -119,6 +119,8 @@ class ValidatorManagerImpl : public ValidatorManager {
   void get_block_proof_link(BlockHandle block_id, td::Promise<td::BufferSlice> promise) override {
     UNREACHABLE();
   }
+  void get_key_block_proof(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) override;
+  void get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) override;
   //void get_block_description(BlockIdExt block_id, td::Promise<BlockDescription> promise) override;
 
   void new_external_message(td::BufferSlice data) override;
@@ -200,11 +202,11 @@ class ValidatorManagerImpl : public ValidatorManager {
   void get_block_proof_link_from_db(BlockHandle handle, td::Promise<td::Ref<ProofLink>> promise) override;
   void get_block_proof_link_from_db_short(BlockIdExt id, td::Promise<td::Ref<ProofLink>> promise) override;
 
-  void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockIdExt> promise) override;
+  void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockHandle> promise) override;
   void get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts,
-                                      td::Promise<BlockIdExt> promise) override;
+                                      td::Promise<BlockHandle> promise) override;
   void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno,
-                                  td::Promise<BlockIdExt> promise) override;
+                                  td::Promise<BlockHandle> promise) override;
 
   // get block handle declared in parent class
   void write_handle(BlockHandle handle, td::Promise<td::Unit> promise) override;
@@ -259,6 +261,14 @@ class ValidatorManagerImpl : public ValidatorManager {
     promise.set_error(td::Status::Error(ErrorCode::error, "download disabled"));
   }
 
+  void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) override {
+    UNREACHABLE();
+  }
+  void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                         td::Promise<td::BufferSlice> promise) override {
+    UNREACHABLE();
+  }
+
   void add_shard_block_description(td::Ref<ShardTopBlockDescription> desc);
 
   void register_block_handle(BlockHandle handle, td::Promise<BlockHandle> promise);
@@ -327,6 +337,9 @@ class ValidatorManagerImpl : public ValidatorManager {
   void allow_block_info_gc(BlockIdExt block_id, td::Promise<bool> promise) override {
     promise.set_result(false);
   }
+  void archive(BlockHandle handle, td::Promise<td::Unit> promise) override {
+    td::actor::send_closure(db_, &Db::archive, std::move(handle), std::move(promise));
+  }
   void update_last_known_key_block(BlockHandle handle, bool send_request) override {
   }
   void update_shard_client_block_handle(BlockHandle handle, td::Promise<td::Unit> promise) override {

validator/manager-init.cpp

@@ -104,16 +104,22 @@ void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice
     return;
   }
 
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+  auto proof_link = pp.move_as_ok();
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof_link](td::Result<BlockHandle> R) {
     if (R.is_error()) {
       LOG(WARNING) << "downloaded proof link failed: " << R.move_as_error();
       td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link);
     } else {
-      td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::try_download_key_blocks, false);
+      auto P = td::PromiseCreator::lambda([SelfId, handle = R.move_as_ok()](td::Result<td::Unit> R) {
+        R.ensure();
+        td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::try_download_key_blocks, false);
+      });
+      td::actor::send_closure(db, &Db::add_key_block_proof_link, proof_link, std::move(P));
     }
   });
 
-  run_check_proof_link_query(handle_->id(), pp.move_as_ok(), manager_, td::Timestamp::in(60.0), std::move(P));
+  run_check_proof_link_query(handle_->id(), proof_link, manager_, td::Timestamp::in(60.0), std::move(P));
 }
 
 void ValidatorManagerMasterchainReiniter::downloaded_zero_state() {
@@ -259,6 +265,8 @@ void ValidatorManagerMasterchainReiniter::download_masterchain_state() {
 
 void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Ref<ShardState> state) {
   state_ = td::Ref<MasterchainState>{std::move(state)};
+  CHECK(handle_->received_state());
+  CHECK(handle_->is_applied());
 
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
     R.ensure();
@@ -452,9 +460,9 @@ void ValidatorManagerMasterchainStarter::got_hardforks(std::vector<BlockIdExt> v
     return;
   }
 
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockIdExt> R) {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
     R.ensure();
-    td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::got_truncate_block_id, R.move_as_ok());
+    td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::got_truncate_block_handle, R.move_as_ok());
   });
   td::actor::send_closure(db_, &Db::get_block_by_seqno, AccountIdPrefixFull{masterchainId, 0}, b.seqno() - 1,
                           std::move(P));

validator/manager.cpp

@@ -29,6 +29,7 @@
 #include "ton/ton-io.hpp"
 #include "state-serializer.hpp"
 #include "get-next-key-blocks.h"
+#include "import-db-slice.hpp"
 
 #include "auto/tl/lite_api.h"
 #include "tl-utils/lite-utils.hpp"
@@ -189,6 +190,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise<Block
     promise.set_error(pp.move_as_error_prefix(PSTRING() << "failed to create block for " << blkid << ": "));
     return;
   }
+  CHECK(blkid.is_masterchain());
 
   auto P = td::PromiseCreator::lambda(
       [SelfId = actor_id(this), promise = std::move(promise), id = block.id](td::Result<td::Unit> R) mutable {
@@ -198,7 +200,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise<Block
           td::actor::send_closure(SelfId, &ValidatorManager::get_block_handle, id, true, std::move(promise));
         }
       });
-  run_apply_block_query(block.id, pp.move_as_ok(), actor_id(this), td::Timestamp::in(10.0), std::move(P));
+  run_apply_block_query(block.id, pp.move_as_ok(), block.id, actor_id(this), td::Timestamp::in(10.0), std::move(P));
 }
 
 void ValidatorManagerImpl::prevalidate_block(BlockBroadcast broadcast, td::Promise<td::Unit> promise) {
@@ -324,6 +326,42 @@ void ValidatorManagerImpl::get_block_proof_link(BlockHandle handle, td::Promise<
   td::actor::send_closure(db_, &Db::get_block_proof_link, std::move(handle), std::move(P));
 }
 
+void ValidatorManagerImpl::get_key_block_proof(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Ref<Proof>> R) mutable {
+    if (R.is_error()) {
+      promise.set_error(R.move_as_error());
+    } else {
+      auto B = R.move_as_ok();
+      promise.set_value(B->data());
+    }
+  });
+
+  td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P));
+}
+
+void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {
+  auto P = td::PromiseCreator::lambda(
+      [promise = std::move(promise), block_id, db = db_.get()](td::Result<td::Ref<Proof>> R) mutable {
+        if (R.is_error()) {
+          auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Ref<Proof>> R) mutable {
+            if (R.is_error()) {
+              promise.set_error(R.move_as_error());
+            } else {
+              auto B = R.move_as_ok();
+              promise.set_value(B->data());
+            }
+          });
+
+          td::actor::send_closure(db, &Db::get_key_block_proof, block_id, std::move(P));
+        } else {
+          auto B = R.move_as_ok()->export_as_proof_link().move_as_ok();
+          promise.set_value(B->data());
+        }
+      });
+
+  td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P));
+}
+
 void ValidatorManagerImpl::new_external_message(td::BufferSlice data) {
   if (!is_validator()) {
     return;
@@ -899,17 +937,17 @@ void ValidatorManagerImpl::get_block_proof_link_from_db_short(BlockIdExt block_i
 }
 
 void ValidatorManagerImpl::get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt,
-                                                   td::Promise<BlockIdExt> promise) {
+                                                   td::Promise<BlockHandle> promise) {
   td::actor::send_closure(db_, &Db::get_block_by_lt, account, lt, std::move(promise));
 }
 
 void ValidatorManagerImpl::get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts,
-                                                          td::Promise<BlockIdExt> promise) {
+                                                          td::Promise<BlockHandle> promise) {
   td::actor::send_closure(db_, &Db::get_block_by_unix_time, account, ts, std::move(promise));
 }
 
 void ValidatorManagerImpl::get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno,
-                                                      td::Promise<BlockIdExt> promise) {
+                                                      td::Promise<BlockHandle> promise) {
   td::actor::send_closure(db_, &Db::get_block_by_seqno, account, seqno, std::move(promise));
 }
 
@@ -1339,6 +1377,7 @@ void ValidatorManagerImpl::start_up() {
   db_ = create_db_actor(actor_id(this), db_root_, opts_->get_filedb_depth());
   lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_);
   token_manager_ = td::actor::create_actor<TokenManager>("tokenmanager");
+  td::mkdir(db_root_ + "/tmp/").ensure();
 
   auto Q =
       td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::actor::ActorOwn<adnl::AdnlExtServer>> R) {
@@ -1375,6 +1414,7 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) {
   gc_masterchain_state_ = std::move(R.gc_state);
 
   shard_client_ = std::move(R.clients);
+  td::actor::send_closure(shard_client_, &ShardClient::start);
 
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::vector<ValidatorSessionId>> R) {
     if (R.is_error()) {
@@ -1741,7 +1781,7 @@ void ValidatorManagerImpl::allow_persistent_state_file_gc(BlockIdExt block_id, B
 }
 
 void ValidatorManagerImpl::allow_archive(BlockIdExt block_id, td::Promise<bool> promise) {
-  if (!gc_masterchain_handle_) {
+  /*if (!gc_masterchain_handle_) {
     promise.set_result(false);
     return;
   }
@@ -1784,7 +1824,8 @@ void ValidatorManagerImpl::allow_archive(BlockIdExt block_id, td::Promise<bool>
       promise.set_result(true);
     }
   });
-  td::actor::send_closure(db_, &Db::archive, block_id, std::move(P));
+  td::actor::send_closure(db_, &Db::archive, block_id, std::move(P));*/
+  promise.set_result(false);
 }
 
 void ValidatorManagerImpl::allow_delete(BlockIdExt block_id, td::Promise<bool> promise) {
@@ -1797,10 +1838,6 @@ void ValidatorManagerImpl::allow_delete(BlockIdExt block_id, td::Promise<bool> p
           return;
         }
         auto handle = R.move_as_ok();
-        if (!handle->moved_to_storage()) {
-          promise.set_result(false);
-          return;
-        }
         if (!handle->inited_unix_time()) {
           promise.set_result(true);
           return;
@@ -1843,15 +1880,13 @@ void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise
 }
 
 void ValidatorManagerImpl::allow_block_info_gc(BlockIdExt block_id, td::Promise<bool> promise) {
-  promise.set_result(false);
-  return;
-  /*auto P =
+  auto P =
       td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
         if (R.is_error()) {
           promise.set_result(false);
         } else {
           auto handle = R.move_as_ok();
-          if (!handle->moved_to_archive()) {
+          if (!handle->moved_to_archive() || !handle->is_applied()) {
             promise.set_result(false);
           } else {
             auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Unit> R) mutable {
@@ -1862,7 +1897,7 @@ void ValidatorManagerImpl::allow_block_info_gc(BlockIdExt block_id, td::Promise<
           }
         }
       });
-  get_block_handle(block_id, false, std::move(P));*/
+  get_block_handle(block_id, false, std::move(P));
 }
 
 void ValidatorManagerImpl::got_next_gc_masterchain_handle(BlockHandle handle) {
@@ -1896,7 +1931,8 @@ void ValidatorManagerImpl::advance_gc(BlockHandle handle, td::Ref<MasterchainSta
 }
 
 void ValidatorManagerImpl::update_shard_client_block_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
-  auto seqno = handle->id().seqno();
+  shard_client_handle_ = std::move(handle);
+  auto seqno = shard_client_handle_->id().seqno();
   shard_client_update(seqno);
   promise.set_value(td::Unit());
 }
@@ -1928,15 +1964,26 @@ void ValidatorManagerImpl::state_serializer_update(BlockSeqno seqno) {
 void ValidatorManagerImpl::alarm() {
   try_advance_gc_masterchain_block();
   alarm_timestamp() = td::Timestamp::in(1.0);
+  if (gc_masterchain_handle_) {
+    td::actor::send_closure(db_, &Db::run_gc, gc_masterchain_handle_->unix_time());
+  }
   if (log_status_at_.is_in_past()) {
     if (last_masterchain_block_handle_) {
       LOG(INFO) << "STATUS: last_masterchain_block_ago="
                 << td::format::as_time(td::Clocks::system() - last_masterchain_block_handle_->unix_time())
                 << " last_known_key_block_ago="
-                << td::format::as_time(td::Clocks::system() - last_known_key_block_handle_->unix_time());
+                << td::format::as_time(td::Clocks::system() - last_known_key_block_handle_->unix_time())
+                << " shard_client_ago="
+                << td::format::as_time(td::Clocks::system() -
+                                       (shard_client_handle_ ? shard_client_handle_->unix_time() : 0));
     }
     log_status_at_ = td::Timestamp::in(60.0);
   }
+  if (false && !downloading_archive_slice_ && shard_client_handle_ &&
+      shard_client_handle_->unix_time() + 600 <= td::Clocks::system() && next_download_archive_slice_at_.is_in_past()) {
+    next_download_archive_slice_at_ = td::Timestamp::in(10.0);
+    try_download_archive_slice();
+  }
   alarm_timestamp().relax(log_status_at_);
   if (resend_shard_blocks_at_ && resend_shard_blocks_at_.is_in_past()) {
     resend_shard_blocks_at_ = td::Timestamp::never();
@@ -1964,17 +2011,6 @@ void ValidatorManagerImpl::alarm() {
   if (check_shard_clients_.is_in_past()) {
     check_shard_clients_ = td::Timestamp::in(10.0);
 
-    if (!shard_client_.empty()) {
-      auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockSeqno> R) {
-        if (R.is_error()) {
-          VLOG(VALIDATOR_WARNING) << "failed to get shard client status: " << R.move_as_error();
-        } else {
-          td::actor::send_closure(SelfId, &ValidatorManagerImpl::shard_client_update, R.move_as_ok());
-        }
-      });
-      td::actor::send_closure(shard_client_, &ShardClient::get_processed_masterchain_block, std::move(P));
-    }
-
     if (!serializer_.empty()) {
       auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockSeqno> R) {
         if (R.is_error()) {
@@ -2017,6 +2053,58 @@ void ValidatorManagerImpl::try_get_static_file(FileHash file_hash, td::Promise<t
   td::actor::send_closure(db_, &Db::try_get_static_file, file_hash, std::move(promise));
 }
 
+void ValidatorManagerImpl::try_download_archive_slice() {
+  CHECK(shard_client_handle_);
+  downloading_archive_slice_ = true;
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::string> R) {
+    if (R.is_error()) {
+      LOG(INFO) << "failed to download archive slice: " << R.error();
+      td::actor::send_closure(SelfId, &ValidatorManagerImpl::failed_to_download_archive_slice);
+    } else {
+      td::actor::send_closure(SelfId, &ValidatorManagerImpl::downloaded_archive_slice, R.move_as_ok());
+    }
+  });
+  callback_->download_archive(shard_client_handle_->id().seqno(), db_root_ + "/tmp/", td::Timestamp::in(3600.0),
+                              std::move(P));
+}
+
+void ValidatorManagerImpl::failed_to_download_archive_slice() {
+  downloading_archive_slice_ = false;
+}
+
+void ValidatorManagerImpl::downloaded_archive_slice(std::string name) {
+  LOG(INFO) << "downloaded archive slice: " << name;
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::vector<BlockSeqno>> R) {
+    if (R.is_error()) {
+      LOG(INFO) << "failed to check downloaded archive slice: " << R.error();
+      td::actor::send_closure(SelfId, &ValidatorManagerImpl::failed_to_download_archive_slice);
+    } else {
+      td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.move_as_ok());
+    }
+  });
+
+  td::actor::create_actor<ArchiveImporter>("archiveimport", name, last_masterchain_state_,
+                                           shard_client_handle_->id().seqno(), opts_, actor_id(this), std::move(P))
+      .release();
+}
+
+void ValidatorManagerImpl::checked_archive_slice(std::vector<BlockSeqno> seqno) {
+  CHECK(seqno.size() == 2);
+  LOG(INFO) << "checked downloaded archive slice: mc_top_seqno=" << seqno[0] << " shard_top_seqno_=" << seqno[1];
+  downloading_archive_slice_ = false;
+  next_download_archive_slice_at_ = td::Timestamp::in(10.0);
+}
+
+void ValidatorManagerImpl::get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) {
+  td::actor::send_closure(db_, &Db::get_archive_id, masterchain_seqno, std::move(promise));
+}
+
+void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                                             td::Promise<td::BufferSlice> promise) {
+  td::actor::send_closure(db_, &Db::get_archive_slice, archive_id, offset, limit, std::move(promise));
+}
+
 bool ValidatorManagerImpl::is_validator() {
   return temp_keys_.size() > 0 || permanent_keys_.size() > 0;
 }

validator/manager.hpp

@@ -236,6 +236,7 @@ class ValidatorManagerImpl : public ValidatorManager {
 
   BlockHandle last_key_block_handle_;
   BlockHandle last_known_key_block_handle_;
+  BlockHandle shard_client_handle_;
 
   BlockHandle gc_masterchain_handle_;
   td::Ref<MasterchainState> gc_masterchain_state_;
@@ -308,6 +309,8 @@ class ValidatorManagerImpl : public ValidatorManager {
                                   td::int64 max_length, td::Promise<td::BufferSlice> promise) override;
   void get_block_proof(BlockHandle handle, td::Promise<td::BufferSlice> promise) override;
   void get_block_proof_link(BlockHandle block_id, td::Promise<td::BufferSlice> promise) override;
+  void get_key_block_proof(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) override;
+  void get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) override;
   //void get_block_description(BlockIdExt block_id, td::Promise<BlockDescription> promise) override;
 
   void new_external_message(td::BufferSlice data) override;
@@ -385,11 +388,11 @@ class ValidatorManagerImpl : public ValidatorManager {
   void get_block_proof_link_from_db(BlockHandle handle, td::Promise<td::Ref<ProofLink>> promise) override;
   void get_block_proof_link_from_db_short(BlockIdExt id, td::Promise<td::Ref<ProofLink>> promise) override;
 
-  void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockIdExt> promise) override;
+  void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockHandle> promise) override;
   void get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts,
-                                      td::Promise<BlockIdExt> promise) override;
+                                      td::Promise<BlockHandle> promise) override;
   void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno,
-                                  td::Promise<BlockIdExt> promise) override;
+                                  td::Promise<BlockHandle> promise) override;
 
   // get block handle declared in parent class
   void write_handle(BlockHandle handle, td::Promise<td::Unit> promise) override;
@@ -424,6 +427,10 @@ class ValidatorManagerImpl : public ValidatorManager {
   void get_async_serializer_state(td::Promise<AsyncSerializerState> promise) override;
 
   void try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice> promise) override;
+  void try_download_archive_slice();
+  void downloaded_archive_slice(std::string name);
+  void checked_archive_slice(std::vector<BlockSeqno> seqno);
+  void failed_to_download_archive_slice();
 
   void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout,
                           td::Promise<std::unique_ptr<DownloadToken>> promise) override {
@@ -431,6 +438,10 @@ class ValidatorManagerImpl : public ValidatorManager {
                             std::move(promise));
   }
 
+  void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) override;
+  void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                         td::Promise<td::BufferSlice> promise) override;
+
   void check_is_hardfork(BlockIdExt block_id, td::Promise<bool> promise) override {
     CHECK(block_id.is_masterchain());
     promise.set_result(opts_->is_hardfork(block_id));
@@ -484,6 +495,9 @@ class ValidatorManagerImpl : public ValidatorManager {
     allow_block_state_gc(block_id, std::move(promise));
   }
   void allow_block_info_gc(BlockIdExt block_id, td::Promise<bool> promise) override;
+  void archive(BlockHandle handle, td::Promise<td::Unit> promise) override {
+    td::actor::send_closure(db_, &Db::archive, std::move(handle), std::move(promise));
+  }
 
   void send_peek_key_block_request();
   void got_next_key_blocks(std::vector<BlockIdExt> vec);
@@ -544,6 +558,9 @@ class ValidatorManagerImpl : public ValidatorManager {
   bool started_ = false;
   bool allow_validate_ = false;
 
+  bool downloading_archive_slice_ = false;
+  td::Timestamp next_download_archive_slice_at_ = td::Timestamp::now();
+
  private:
   double state_ttl() const {
     return opts_->state_ttl();

validator/net/download-archive-slice.cpp

@@ -0,0 +1,177 @@
+#include "download-archive-slice.hpp"
+#include "td/utils/port/path.h"
+#include "td/utils/overloaded.h"
+
+namespace ton {
+
+namespace validator {
+
+namespace fullnode {
+
+DownloadArchiveSlice::DownloadArchiveSlice(
+    BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id,
+    overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout,
+    td::actor::ActorId<ValidatorManagerInterface> validator_manager, td::actor::ActorId<rldp::Rldp> rldp,
+    td::actor::ActorId<overlay::Overlays> overlays, td::actor::ActorId<adnl::Adnl> adnl,
+    td::actor::ActorId<adnl::AdnlExtClient> client, td::Promise<std::string> promise)
+    : masterchain_seqno_(masterchain_seqno)
+    , tmp_dir_(std::move(tmp_dir))
+    , local_id_(local_id)
+    , overlay_id_(overlay_id)
+    , download_from_(download_from)
+    , timeout_(timeout)
+    , validator_manager_(validator_manager)
+    , rldp_(rldp)
+    , overlays_(overlays)
+    , adnl_(adnl)
+    , client_(client)
+    , promise_(std::move(promise)) {
+}
+
+void DownloadArchiveSlice::abort_query(td::Status reason) {
+  if (promise_) {
+    promise_.set_error(std::move(reason));
+    if (!fd_.empty()) {
+      td::unlink(tmp_name_).ensure();
+      fd_.close();
+    }
+  }
+  stop();
+}
+
+void DownloadArchiveSlice::alarm() {
+  abort_query(td::Status::Error(ErrorCode::timeout, "timeout"));
+}
+
+void DownloadArchiveSlice::finish_query() {
+  if (promise_) {
+    promise_.set_value(std::move(tmp_name_));
+    fd_.close();
+  }
+  stop();
+}
+
+void DownloadArchiveSlice::start_up() {
+  alarm_timestamp() = timeout_;
+
+  auto R = td::mkstemp(tmp_dir_);
+  if (R.is_error()) {
+    abort_query(R.move_as_error_prefix("failed to open temp file: "));
+    return;
+  }
+  auto r = R.move_as_ok();
+  fd_ = std::move(r.first);
+  tmp_name_ = std::move(r.second);
+
+  if (download_from_.is_zero() && client_.empty()) {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::vector<adnl::AdnlNodeIdShort>> R) {
+      if (R.is_error()) {
+        td::actor::send_closure(SelfId, &DownloadArchiveSlice::abort_query, R.move_as_error());
+      } else {
+        auto vec = R.move_as_ok();
+        if (vec.size() == 0) {
+          td::actor::send_closure(SelfId, &DownloadArchiveSlice::abort_query,
+                                  td::Status::Error(ErrorCode::notready, "no nodes"));
+        } else {
+          td::actor::send_closure(SelfId, &DownloadArchiveSlice::got_node_to_download, vec[0]);
+        }
+      }
+    });
+
+    td::actor::send_closure(overlays_, &overlay::Overlays::get_overlay_random_peers, local_id_, overlay_id_, 1,
+                            std::move(P));
+  } else {
+    got_node_to_download(download_from_);
+  }
+}
+
+void DownloadArchiveSlice::got_node_to_download(adnl::AdnlNodeIdShort download_from) {
+  download_from_ = download_from;
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &DownloadArchiveSlice::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &DownloadArchiveSlice::got_archive_info, R.move_as_ok());
+    }
+  });
+
+  auto q = create_serialize_tl_object<ton_api::tonNode_getArchiveInfo>(masterchain_seqno_);
+  if (client_.empty()) {
+    td::actor::send_closure(overlays_, &overlay::Overlays::send_query, download_from_, local_id_, overlay_id_,
+                            "get_archive_info", std::move(P), td::Timestamp::in(3.0), std::move(q));
+  } else {
+    td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_archive_info",
+                            create_serialize_tl_object_suffix<ton_api::tonNode_query>(std::move(q)),
+                            td::Timestamp::in(1.0), std::move(P));
+  }
+}
+
+void DownloadArchiveSlice::got_archive_info(td::BufferSlice data) {
+  auto F = fetch_tl_object<ton_api::tonNode_ArchiveInfo>(std::move(data), true);
+  if (F.is_error()) {
+    abort_query(F.move_as_error_prefix("failed to parse ArchiveInfo answer"));
+    return;
+  }
+  auto f = F.move_as_ok();
+
+  bool fail = false;
+  ton_api::downcast_call(*f.get(), td::overloaded(
+                                       [&](const ton_api::tonNode_archiveNotFound &obj) {
+                                         abort_query(td::Status::Error(ErrorCode::notready, "remote db not found"));
+                                         fail = true;
+                                       },
+                                       [&](const ton_api::tonNode_archiveInfo &obj) { archive_id_ = obj.id_; }));
+  if (fail) {
+    return;
+  }
+
+  get_archive_slice();
+}
+
+void DownloadArchiveSlice::get_archive_slice() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &DownloadArchiveSlice::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &DownloadArchiveSlice::got_archive_slice, R.move_as_ok());
+    }
+  });
+
+  auto q = create_serialize_tl_object<ton_api::tonNode_getArchiveSlice>(archive_id_, offset_, slice_size());
+  if (client_.empty()) {
+    td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_,
+                            "get_archive_slice", std::move(P), td::Timestamp::in(3.0), std::move(q),
+                            slice_size() + 1024, rldp_);
+  } else {
+    td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_archive_slice",
+                            create_serialize_tl_object_suffix<ton_api::tonNode_query>(std::move(q)),
+                            td::Timestamp::in(1.0), std::move(P));
+  }
+}
+
+void DownloadArchiveSlice::got_archive_slice(td::BufferSlice data) {
+  auto R = fd_.write(data.as_slice());
+  if (R.is_error()) {
+    abort_query(R.move_as_error_prefix("failed to write temp file: "));
+    return;
+  }
+  if (R.move_as_ok() != data.size()) {
+    abort_query(td::Status::Error(ErrorCode::error, "short write to temp file"));
+    return;
+  }
+
+  offset_ += data.size();
+
+  if (data.size() < slice_size()) {
+    finish_query();
+  } else {
+    get_archive_slice();
+  }
+}
+
+}  // namespace fullnode
+
+}  // namespace validator
+
+}  // namespace ton

validator/net/download-archive-slice.hpp

@@ -0,0 +1,83 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "overlay/overlays.h"
+#include "ton/ton-types.h"
+#include "validator/validator.h"
+#include "rldp/rldp.h"
+#include "adnl/adnl-ext-client.h"
+#include "td/utils/port/FileFd.h"
+
+namespace ton {
+
+namespace validator {
+
+namespace fullnode {
+
+class DownloadArchiveSlice : public td::actor::Actor {
+ public:
+  DownloadArchiveSlice(BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id,
+                       overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout,
+                       td::actor::ActorId<ValidatorManagerInterface> validator_manager,
+                       td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<overlay::Overlays> overlays,
+                       td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<adnl::AdnlExtClient> client,
+                       td::Promise<std::string> promise);
+
+  void abort_query(td::Status reason);
+  void alarm() override;
+  void finish_query();
+
+  void start_up() override;
+  void got_node_to_download(adnl::AdnlNodeIdShort node);
+  void got_archive_info(td::BufferSlice data);
+  void get_archive_slice();
+  void got_archive_slice(td::BufferSlice data);
+
+  static constexpr td::uint32 slice_size() {
+    return 1 << 17;
+  }
+
+ private:
+  BlockSeqno masterchain_seqno_;
+  std::string tmp_dir_;
+  std::string tmp_name_;
+  td::FileFd fd_;
+  adnl::AdnlNodeIdShort local_id_;
+  overlay::OverlayIdShort overlay_id_;
+  td::uint64 offset_ = 0;
+  td::uint64 archive_id_;
+
+  adnl::AdnlNodeIdShort download_from_ = adnl::AdnlNodeIdShort::zero();
+
+  td::Timestamp timeout_;
+  td::actor::ActorId<ValidatorManagerInterface> validator_manager_;
+  td::actor::ActorId<rldp::Rldp> rldp_;
+  td::actor::ActorId<overlay::Overlays> overlays_;
+  td::actor::ActorId<adnl::Adnl> adnl_;
+  td::actor::ActorId<adnl::AdnlExtClient> client_;
+  td::Promise<std::string> promise_;
+};
+
+}  // namespace fullnode
+
+}  // namespace validator
+
+}  // namespace ton
+

validator/net/download-proof.cpp

@@ -79,6 +79,33 @@ void DownloadProof::finish_query() {
 void DownloadProof::start_up() {
   alarm_timestamp() = timeout_;
 
+  if (!block_id_.is_masterchain()) {
+    checked_db();
+    return;
+  }
+
+  auto P =
+      td::PromiseCreator::lambda([SelfId = actor_id(this), l = allow_partial_proof_](td::Result<td::BufferSlice> R) {
+        if (R.is_error()) {
+          td::actor::send_closure(SelfId, &DownloadProof::checked_db);
+        } else {
+          if (l) {
+            td::actor::send_closure(SelfId, &DownloadProof::got_block_partial_proof, R.move_as_ok());
+          } else {
+            td::actor::send_closure(SelfId, &DownloadProof::got_block_proof, R.move_as_ok());
+          }
+        }
+      });
+  if (allow_partial_proof_) {
+    td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof_link, block_id_,
+                            std::move(P));
+  } else {
+    td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof, block_id_,
+                            std::move(P));
+  }
+}
+
+void DownloadProof::checked_db() {
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::unique_ptr<DownloadToken>> R) {
     if (R.is_error()) {
       td::actor::send_closure(SelfId, &DownloadProof::abort_query,

validator/net/download-proof.hpp

@@ -44,6 +44,7 @@ class DownloadProof : public td::actor::Actor {
   void finish_query();
 
   void start_up() override;
+  void checked_db();
   void got_download_token(std::unique_ptr<DownloadToken> token);
   void got_node_to_download(adnl::AdnlNodeIdShort node);
   void got_block_proof_description(td::BufferSlice proof_description);

validator/shard-client.cpp

@@ -39,6 +39,15 @@ void ShardClient::start_up() {
   td::actor::send_closure(manager_, &ValidatorManager::get_shard_client_state, true, std::move(P));
 }
 
+void ShardClient::start() {
+  if (!started_ && masterchain_state_.not_null()) {
+    started_ = true;
+    apply_all_shards();
+  } else {
+    started_ = true;
+  }
+}
+
 void ShardClient::got_state_from_db(BlockIdExt state) {
   CHECK(!init_mode_);
 
@@ -133,7 +142,9 @@ void ShardClient::download_masterchain_state() {
 void ShardClient::got_masterchain_block_state(td::Ref<MasterchainState> state) {
   masterchain_state_ = std::move(state);
   build_shard_overlays();
-  apply_all_shards();
+  if (started_) {
+    apply_all_shards();
+  }
 }
 
 void ShardClient::apply_all_shards() {
@@ -170,8 +181,8 @@ void ShardClient::apply_all_shards() {
 }
 
 void ShardClient::downloaded_shard_state(td::Ref<ShardState> state, td::Promise<td::Unit> promise) {
-  run_apply_block_query(state->get_block_id(), td::Ref<BlockData>{}, manager_, td::Timestamp::in(600),
-                        std::move(promise));
+  run_apply_block_query(state->get_block_id(), td::Ref<BlockData>{}, masterchain_block_handle_->id(), manager_,
+                        td::Timestamp::in(600), std::move(promise));
 }
 
 void ShardClient::new_masterchain_block_notification(BlockHandle handle, td::Ref<MasterchainState> state) {

validator/shard-client.hpp

@@ -36,6 +36,7 @@ class ShardClient : public td::actor::Actor {
 
   bool waiting_ = false;
   bool init_mode_ = false;
+  bool started_ = false;
 
   td::actor::ActorId<ValidatorManager> manager_;
 
@@ -67,6 +68,7 @@ class ShardClient : public td::actor::Actor {
   void start_up() override;
   void start_up_init_mode();
   void start_up_init_mode_finished();
+  void start();
   void got_state_from_db(BlockIdExt masterchain_block_id);
 
   void im_download_shard_state(BlockIdExt block_id, td::Promise<td::Unit> promise);

validator/validate-broadcast.cpp

@@ -110,12 +110,12 @@ void ValidateBroadcast::start_up() {
     } else if (key_block_seqno == last_masterchain_state_->get_seqno()) {
       got_key_block_handle(last_masterchain_block_handle_);
     } else {
-      auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockIdExt> R) {
+      auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
         if (R.is_error()) {
           td::actor::send_closure(SelfId, &ValidateBroadcast::abort_query,
                                   R.move_as_error_prefix("cannot find reference key block id: "));
         } else {
-          td::actor::send_closure(SelfId, &ValidateBroadcast::got_key_block_id, R.move_as_ok());
+          td::actor::send_closure(SelfId, &ValidateBroadcast::got_key_block_handle, R.move_as_ok());
         }
       });
       td::actor::send_closure(manager_, &ValidatorManager::get_block_by_seqno_from_db,
@@ -305,7 +305,9 @@ void ValidateBroadcast::checked_proof() {
       }
     });
 
-    td::actor::create_actor<ApplyBlock>("applyblock", handle_->id(), data_, manager_, timeout_, std::move(P)).release();
+    td::actor::create_actor<ApplyBlock>("applyblock", handle_->id(), data_, handle_->id(), manager_, timeout_,
+                                        std::move(P))
+        .release();
   } else {
     finish_query();
   }

validator/validator.h

@@ -117,6 +117,8 @@ class ValidatorManagerInterface : public td::actor::Actor {
                                            td::Promise<td::BufferSlice> promise) = 0;
     virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout,
                                      td::Promise<std::vector<BlockIdExt>> promise) = 0;
+    virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                                  td::Promise<std::string> promise) = 0;
 
     virtual void new_key_block(BlockHandle handle) = 0;
   };
@@ -157,6 +159,8 @@ class ValidatorManagerInterface : public td::actor::Actor {
   virtual void get_block_proof(BlockHandle handle, td::Promise<td::BufferSlice> promise) = 0;
   virtual void get_block_proof_link(BlockHandle handle, td::Promise<td::BufferSlice> promise) = 0;
   virtual void get_block_handle(BlockIdExt block_id, bool force, td::Promise<BlockHandle> promise) = 0;
+  virtual void get_key_block_proof(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) = 0;
+  virtual void get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) = 0;
   virtual void get_next_key_blocks(BlockIdExt block_id, td::uint32 cnt,
                                    td::Promise<std::vector<BlockIdExt>> promise) = 0;
   virtual void get_next_block(BlockIdExt block_id, td::Promise<BlockHandle> promise) = 0;
@@ -184,11 +188,15 @@ class ValidatorManagerInterface : public td::actor::Actor {
   virtual void get_block_proof_link_from_db_short(BlockIdExt id, td::Promise<td::Ref<ProofLink>> promise) = 0;
 
   virtual void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt,
-                                       td::Promise<BlockIdExt> promise) = 0;
+                                       td::Promise<BlockHandle> promise) = 0;
   virtual void get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts,
-                                              td::Promise<BlockIdExt> promise) = 0;
+                                              td::Promise<BlockHandle> promise) = 0;
   virtual void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno,
-                                          td::Promise<BlockIdExt> promise) = 0;
+                                          td::Promise<BlockHandle> promise) = 0;
+
+  virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) = 0;
+  virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                                 td::Promise<td::BufferSlice> promise) = 0;
 
   virtual void run_ext_query(td::BufferSlice data, td::Promise<td::BufferSlice> promise) = 0;
   virtual void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) = 0;