updated submodules, bugfixes

- added new fift/func code for validator complaint creation
- bugfixes in validator
- updates in tonlib
- new versions of rocksdb/abseil
- hardfork support

validator/db/archive-manager.cpp

@@ -77,10 +77,20 @@ void ArchiveManager::update_handle(BlockHandle handle, td::Promise<td::Unit> pro
   FileDescription *f;
   if (handle->handle_moved_to_archive()) {
     CHECK(handle->inited_unix_time());
+    if (!handle->need_flush()) {
+      promise.set_value(td::Unit());
+      return;
+    }
     f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), handle->id().seqno(),
                       handle->unix_time(), handle->logical_time(), true);
+    if (!f) {
+      handle->flushed_upto(handle->version());
+      promise.set_value(td::Unit());
+      return;
+    }
   } else {
     f = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true);
+    CHECK(f);
   }
   td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::update_handle, std::move(handle), std::move(promise));
 }
@@ -540,7 +550,7 @@ void ArchiveManager::load_package(PackageId id) {
     }
   }
 
-  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, db_root_);
+  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_);
 
   get_file_map(id).emplace(id, std::move(desc));
 }
@@ -551,7 +561,6 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard
   auto it = f.find(id);
   if (it != f.end()) {
     if (it->second.deleted) {
-      CHECK(!force);
       return nullptr;
     }
     if (force && !id.temp) {
@@ -574,7 +583,7 @@ ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard
   FileDescription desc{id, false};
   td::mkdir(db_root_ + id.path()).ensure();
   std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
-  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, db_root_);
+  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_);
   if (!id.temp) {
     update_desc(desc, shard, seqno, ts, lt);
   }
@@ -648,7 +657,11 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdF
   for (auto it = f.rbegin(); it != f.rend(); it++) {
     auto i = it->second.first_blocks.find(shard);
     if (i != it->second.first_blocks.end() && i->second.seqno <= seqno) {
-      return &it->second;
+      if (it->second.deleted) {
+        return nullptr;
+      } else {
+        return &it->second;
+      }
     }
   }
   return nullptr;
@@ -660,7 +673,11 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(Shar
   for (auto it = f.rbegin(); it != f.rend(); it++) {
     auto i = it->second.first_blocks.find(shard);
     if (i != it->second.first_blocks.end() && i->second.ts <= ts) {
-      return &it->second;
+      if (it->second.deleted) {
+        return nullptr;
+      } else {
+        return &it->second;
+      }
     }
   }
   return nullptr;
@@ -672,7 +689,11 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull
   for (auto it = f.rbegin(); it != f.rend(); it++) {
     auto i = it->second.first_blocks.find(shard);
     if (i != it->second.first_blocks.end() && i->second.lt <= lt) {
-      return &it->second;
+      if (it->second.deleted) {
+        return nullptr;
+      } else {
+        return &it->second;
+      }
     }
   }
   return nullptr;
@@ -823,6 +844,14 @@ void ArchiveManager::start_up() {
     }
   }
 
+  v = index_->get("finalizedupto", value);
+  v.ensure();
+  if (v.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+    auto R = td::to_integer_safe<td::uint32>(value);
+    R.ensure();
+    finalized_up_to_ = R.move_as_ok();
+  }
+
   td::WalkPath::run(db_root_ + "/archive/states/", [&](td::CSlice fname, td::WalkPath::Type t) -> void {
     if (t == td::WalkPath::Type::NotDir) {
       LOG(ERROR) << "checking file " << fname;
@@ -853,7 +882,7 @@ void ArchiveManager::start_up() {
   persistent_state_gc(FileHash::zero());
 }
 
-void ArchiveManager::run_gc(UnixTime ts) {
+void ArchiveManager::run_gc(UnixTime ts, UnixTime archive_ttl) {
   auto p = get_temp_package_id_by_unixtime(ts);
   std::vector<PackageId> vec;
   for (auto &x : temp_files_) {
@@ -863,13 +892,37 @@ void ArchiveManager::run_gc(UnixTime ts) {
       break;
     }
   }
-  if (vec.size() <= 1) {
-    return;
-  }
-  vec.resize(vec.size() - 1, PackageId::empty(false, true));
+  if (vec.size() > 1) {
+    vec.resize(vec.size() - 1, PackageId::empty(false, true));
 
-  for (auto &x : vec) {
-    delete_package(x, [](td::Unit) {});
+    for (auto &x : vec) {
+      delete_package(x, [](td::Unit) {});
+    }
+  }
+  vec.clear();
+
+  if (archive_ttl > 0) {
+    for (auto &f : files_) {
+      auto &desc = f.second;
+      if (desc.deleted) {
+        continue;
+      }
+      auto it = desc.first_blocks.find(ShardIdFull{masterchainId});
+      if (it == desc.first_blocks.end()) {
+        continue;
+      }
+      if (it->second.ts < ts - archive_ttl) {
+        vec.push_back(f.first);
+      }
+    }
+    if (vec.size() > 1) {
+      vec.resize(vec.size() - 1, PackageId::empty(false, true));
+
+      for (auto &x : vec) {
+        LOG(ERROR) << "WARNING: deleting package " << x.id;
+        delete_package(x, [](td::Unit) {});
+      }
+    }
   }
 }
 
@@ -1049,6 +1102,102 @@ void ArchiveManager::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
   }
 }
 
+void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise) {
+  index_->begin_transaction().ensure();
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise(std::move(promise));
+  for (auto &x : temp_files_) {
+    if (!x.second.deleted) {
+      td::actor::send_closure(x.second.file_actor_id(), &ArchiveSlice::destroy, ig.get_promise());
+      x.second.file.release();
+    }
+  }
+  temp_files_.clear();
+
+  {
+    auto it = key_files_.begin();
+    while (it != key_files_.end()) {
+      if (it->first.id <= masterchain_seqno) {
+        td::actor::send_closure(it->second.file_actor_id(), &ArchiveSlice::truncate, masterchain_seqno, handle,
+                                ig.get_promise());
+        it++;
+      } else {
+        auto it2 = it;
+        it++;
+        td::actor::send_closure(it2->second.file_actor_id(), &ArchiveSlice::destroy, ig.get_promise());
+        it2->second.file.release();
+        index_
+            ->erase(create_serialize_tl_object<ton_api::db_files_package_key>(it2->second.id.id, it2->second.id.key,
+                                                                              it2->second.id.temp)
+                        .as_slice())
+            .ensure();
+        key_files_.erase(it2);
+      }
+    }
+  }
+  {
+    auto it = files_.begin();
+    while (it != files_.end()) {
+      if (it->first.id <= masterchain_seqno) {
+        td::actor::send_closure(it->second.file_actor_id(), &ArchiveSlice::truncate, masterchain_seqno, handle,
+                                ig.get_promise());
+        it++;
+      } else {
+        auto it2 = it;
+        it++;
+        td::actor::send_closure(it2->second.file_actor_id(), &ArchiveSlice::destroy, ig.get_promise());
+        it2->second.file.release();
+        index_
+            ->erase(create_serialize_tl_object<ton_api::db_files_package_key>(it2->second.id.id, it2->second.id.key,
+                                                                              it2->second.id.temp)
+                        .as_slice())
+            .ensure();
+        files_.erase(it2);
+      }
+    }
+  }
+  {
+    std::vector<td::int32> t;
+    std::vector<td::int32> tk;
+    std::vector<td::int32> tt;
+    for (auto &e : files_) {
+      t.push_back(e.first.id);
+    }
+    for (auto &e : key_files_) {
+      tk.push_back(e.first.id);
+    }
+    for (auto &e : temp_files_) {
+      tt.push_back(e.first.id);
+    }
+    index_
+        ->set(create_serialize_tl_object<ton_api::db_files_index_key>().as_slice(),
+              create_serialize_tl_object<ton_api::db_files_index_value>(std::move(t), std::move(tk), std::move(tt))
+                  .as_slice())
+        .ensure();
+  }
+  index_->commit_transaction().ensure();
+
+  {
+    auto it = perm_states_.begin();
+    while (it != perm_states_.end()) {
+      int res = 0;
+      it->second.ref().visit(td::overloaded(
+          [&](const fileref::ZeroStateShort &x) { res = -1; },
+          [&](const fileref::PersistentStateShort &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; },
+          [&](const auto &obj) { res = 1; }));
+      if (res <= 0) {
+        it++;
+      } else {
+        auto it2 = it;
+        it++;
+        td::unlink(db_root_ + "/archive/states/" + it2->second.filename_short()).ignore();
+        perm_states_.erase(it2);
+      }
+    }
+  }
+}
+
 }  // namespace validator
 
 }  // namespace ton

validator/db/archive-manager.hpp

@@ -52,10 +52,10 @@ class ArchiveManager : public td::actor::Actor {
   void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<bool> promise);
   void check_zero_state(BlockIdExt block_id, td::Promise<bool> promise);
 
-  //void truncate(BlockSeqno masterchain_seqno, td::Promise<td::Unit> promise);
+  void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise);
   //void truncate_continue(BlockSeqno masterchain_seqno, td::Promise<td::Unit> promise);
 
-  void run_gc(UnixTime ts);
+  void run_gc(UnixTime ts, UnixTime archive_ttl);
 
   /* from LTDB */
   void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise<ConstBlockHandle> promise);
@@ -104,6 +104,7 @@ class ArchiveManager : public td::actor::Actor {
   std::map<PackageId, FileDescription> files_;
   std::map<PackageId, FileDescription> key_files_;
   std::map<PackageId, FileDescription> temp_files_;
+  BlockSeqno finalized_up_to_{0};
   bool async_mode_ = false;
   bool huge_transaction_started_ = false;
   td::uint32 huge_transaction_size_ = 0;

validator/db/archive-slice.cpp

@@ -309,7 +309,7 @@ void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id,
       }
     }
     f = true;
-    auto G = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
+    auto G = fetch_tl_object<ton_api::db_lt_desc_value>(value, true);
     G.ensure();
     auto g = G.move_as_ok();
     if (compare_desc(*g.get()) > 0) {
@@ -467,12 +467,18 @@ void ArchiveSlice::start_up() {
         R2 = kv_->get(PSTRING() << "status." << i, value);
         R2.ensure();
         auto len = td::to_integer<td::uint64>(value);
+        R2 = kv_->get(PSTRING() << "version." << i, value);
+        R2.ensure();
+        td::uint32 ver = 0;
+        if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+          ver = td::to_integer<td::uint32>(value);
+        }
         auto v = archive_id_ + slice_size_ * i;
-        add_package(v, len);
+        add_package(v, len, ver);
       }
     } else {
       auto len = td::to_integer<td::uint64>(value);
-      add_package(archive_id_, len);
+      add_package(archive_id_, len, 0);
     }
   } else {
     if (!temp_ && !key_blocks_only_) {
@@ -482,13 +488,15 @@ void ArchiveSlice::start_up() {
       kv_->set("slices", "1").ensure();
       kv_->set("slice_size", td::to_string(slice_size_)).ensure();
       kv_->set("status.0", "0").ensure();
+      kv_->set("version.0", td::to_string(default_package_version())).ensure();
       kv_->commit_transaction().ensure();
+      add_package(archive_id_, 0, default_package_version());
     } else {
       kv_->begin_transaction().ensure();
       kv_->set("status", "0").ensure();
       kv_->commit_transaction().ensure();
+      add_package(archive_id_, 0, 0);
     }
-    add_package(archive_id_, 0);
   }
 }
 
@@ -528,8 +536,12 @@ void ArchiveSlice::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
   }
 }
 
-ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, std::string db_root)
-    : archive_id_(archive_id), key_blocks_only_(key_blocks_only), temp_(temp), db_root_(std::move(db_root)) {
+ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root)
+    : archive_id_(archive_id)
+    , key_blocks_only_(key_blocks_only)
+    , temp_(temp)
+    , finalized_(finalized)
+    , db_root_(std::move(db_root)) {
 }
 
 td::Result<ArchiveSlice::PackageInfo *> ArchiveSlice::choose_package(BlockSeqno masterchain_seqno, bool force) {
@@ -548,16 +560,17 @@ td::Result<ArchiveSlice::PackageInfo *> ArchiveSlice::choose_package(BlockSeqno
     begin_transaction();
     kv_->set("slices", td::to_string(v + 1)).ensure();
     kv_->set(PSTRING() << "status." << v, "0").ensure();
+    kv_->set(PSTRING() << "version." << v, td::to_string(default_package_version())).ensure();
     commit_transaction();
     CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0);
-    add_package(masterchain_seqno, 0);
+    add_package(masterchain_seqno, 0, default_package_version());
     return &packages_[v];
   } else {
     return &packages_[v];
   }
 }
 
-void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size) {
+void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 version) {
   PackageId p_id{seqno, key_blocks_only_, temp_};
   std::string path = PSTRING() << db_root_ << p_id.path() << p_id.name() << ".pack";
   auto R = Package::open(path, false, true);
@@ -565,9 +578,17 @@ void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size) {
     LOG(FATAL) << "failed to open/create archive '" << path << "': " << R.move_as_error();
     return;
   }
+  auto idx = td::narrow_cast<td::uint32>(packages_.size());
+  if (finalized_) {
+    packages_.emplace_back(nullptr, td::actor::ActorOwn<PackageWriter>(), seqno, path, idx, version);
+    return;
+  }
   auto pack = std::make_shared<Package>(R.move_as_ok());
+  if (version >= 1) {
+    pack->truncate(size).ensure();
+  }
   auto writer = td::actor::create_actor<PackageWriter>("writer", pack);
-  packages_.emplace_back(std::move(pack), std::move(writer), seqno, path, 0);
+  packages_.emplace_back(std::move(pack), std::move(writer), seqno, path, idx, version);
 }
 
 namespace {
@@ -609,6 +630,207 @@ void ArchiveSlice::destroy(td::Promise<td::Unit> promise) {
                td::Timestamp::in(0.0));
 }
 
+BlockSeqno ArchiveSlice::max_masterchain_seqno() {
+  auto key = get_db_key_lt_desc(ShardIdFull{masterchainId});
+  std::string value;
+  auto F = kv_->get(key, value);
+  F.ensure();
+  if (F.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+    return 0;
+  }
+  auto G = fetch_tl_object<ton_api::db_lt_desc_value>(value, true);
+  G.ensure();
+  auto g = G.move_as_ok();
+  if (g->first_idx_ == g->last_idx_) {
+    return 0;
+  }
+  auto last_idx = g->last_idx_ - 1;
+  auto db_key = get_db_key_lt_el(ShardIdFull{masterchainId}, last_idx);
+  F = kv_->get(db_key, value);
+  F.ensure();
+  CHECK(F.move_as_ok() == td::KeyValue::GetStatus::Ok);
+  auto E = fetch_tl_object<ton_api::db_lt_el_value>(td::BufferSlice{value}, true);
+  E.ensure();
+  auto e = E.move_as_ok();
+  return e->id_->seqno_;
+}
+
+void ArchiveSlice::delete_file(FileReference ref_id) {
+  std::string value;
+  auto R = kv_->get(ref_id.hash().to_hex(), value);
+  R.ensure();
+  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+    return;
+  }
+  kv_->erase(ref_id.hash().to_hex());
+}
+
+void ArchiveSlice::delete_handle(ConstBlockHandle handle) {
+  delete_file(fileref::Proof{handle->id()});
+  delete_file(fileref::ProofLink{handle->id()});
+  delete_file(fileref::Block{handle->id()});
+  kv_->erase(get_db_key_block_info(handle->id()));
+}
+
+void ArchiveSlice::move_file(FileReference ref_id, Package *old_pack, Package *pack) {
+  LOG(DEBUG) << "moving " << ref_id.filename_short();
+  std::string value;
+  auto R = kv_->get(ref_id.hash().to_hex(), value);
+  R.ensure();
+  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+    return;
+  }
+  auto offset = td::to_integer<td::uint64>(value);
+  auto V = old_pack->read(offset);
+  V.ensure();
+  auto data = std::move(V.move_as_ok().second);
+  auto r = pack->append(ref_id.filename(), std::move(data), false);
+  kv_->set(ref_id.hash().to_hex(), td::to_string(r));
+}
+
+void ArchiveSlice::move_handle(ConstBlockHandle handle, Package *old_pack, Package *pack) {
+  move_file(fileref::Proof{handle->id()}, old_pack, pack);
+  move_file(fileref::ProofLink{handle->id()}, old_pack, pack);
+  move_file(fileref::Block{handle->id()}, old_pack, pack);
+}
+
+bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx,
+                                  Package *pack) {
+  std::string value;
+  auto R = kv_->get(get_db_key_block_info(block_id), value);
+  R.ensure();
+  CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok);
+  auto E = create_block_handle(value);
+  E.ensure();
+  auto handle = E.move_as_ok();
+  auto seqno = handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block();
+  if (seqno > masterchain_seqno) {
+    delete_handle(std::move(handle));
+    return false;
+  }
+
+  auto S = choose_package(seqno, false);
+  S.ensure();
+  auto p = S.move_as_ok();
+  CHECK(p->idx <= cutoff_idx);
+  if (p->idx == cutoff_idx) {
+    move_handle(std::move(handle), p->package.get(), pack);
+  }
+
+  return true;
+}
+
+void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx,
+                                  Package *pack) {
+  auto key = get_db_key_lt_desc(shard);
+  std::string value;
+  auto F = kv_->get(key, value);
+  F.ensure();
+  if (F.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+    return;
+  }
+  auto G = fetch_tl_object<ton_api::db_lt_desc_value>(value, true);
+  G.ensure();
+  auto g = G.move_as_ok();
+  if (g->first_idx_ == g->last_idx_) {
+    return;
+  }
+
+  int new_last_idx = g->first_idx_;
+  for (int i = g->first_idx_; i < g->last_idx_; i++) {
+    auto db_key = get_db_key_lt_el(shard, i);
+    F = kv_->get(db_key, value);
+    F.ensure();
+    CHECK(F.move_as_ok() == td::KeyValue::GetStatus::Ok);
+    auto E = fetch_tl_object<ton_api::db_lt_el_value>(value, true);
+    E.ensure();
+    auto e = E.move_as_ok();
+
+    if (truncate_block(masterchain_seqno, create_block_id(e->id_), cutoff_idx, pack)) {
+      CHECK(new_last_idx == i);
+      new_last_idx = i + 1;
+    }
+  }
+
+  if (g->last_idx_ != new_last_idx) {
+    g->last_idx_ = new_last_idx;
+    kv_->set(key, serialize_tl_object(g, true)).ensure();
+  }
+}
+
+void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise) {
+  if (temp_ || archive_id_ > masterchain_seqno) {
+    destroy(std::move(promise));
+    return;
+  }
+  LOG(INFO) << "TRUNCATE: slice " << archive_id_ << " maxseqno= " << max_masterchain_seqno()
+            << " truncate_upto=" << masterchain_seqno;
+  if (max_masterchain_seqno() <= masterchain_seqno) {
+    promise.set_value(td::Unit());
+    return;
+  }
+
+  auto cutoff = choose_package(masterchain_seqno, false);
+  cutoff.ensure();
+  auto pack = cutoff.move_as_ok();
+  CHECK(pack);
+
+  auto pack_r = Package::open(pack->path + ".new", false, true);
+  pack_r.ensure();
+  auto new_package = std::make_shared<Package>(pack_r.move_as_ok());
+  new_package->truncate(0).ensure();
+
+  std::string value;
+  auto status_key = create_serialize_tl_object<ton_api::db_lt_status_key>();
+  auto R = kv_->get(status_key, value);
+  R.ensure();
+
+  auto F = fetch_tl_object<ton_api::db_lt_status_value>(value, true);
+  F.ensure();
+  auto f = F.move_as_ok();
+
+  kv_->begin_transaction().ensure();
+  for (int i = 0; i < f->total_shards_; i++) {
+    auto shard_key = create_serialize_tl_object<ton_api::db_lt_shard_key>(i);
+    R = kv_->get(shard_key, value);
+    R.ensure();
+    CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok);
+
+    auto G = fetch_tl_object<ton_api::db_lt_shard_value>(value, true);
+    G.ensure();
+    auto g = G.move_as_ok();
+
+    truncate_shard(masterchain_seqno, ShardIdFull{g->workchain_, static_cast<td::uint64>(g->shard_)}, pack->idx,
+                   new_package.get());
+  }
+
+  if (!sliced_mode_) {
+    kv_->set("status", td::to_string(new_package->size())).ensure();
+  } else {
+    kv_->set(PSTRING() << "status." << pack->idx, td::to_string(new_package->size())).ensure();
+    for (size_t i = pack->idx + 1; i < packages_.size(); i++) {
+      kv_->erase(PSTRING() << "status." << i);
+      kv_->erase(PSTRING() << "version." << i);
+    }
+    kv_->set("slices", td::to_string(pack->idx + 1));
+  }
+
+  pack->package = new_package;
+  pack->writer.reset();
+  td::unlink(pack->path).ensure();
+  td::rename(pack->path + ".new", pack->path).ensure();
+  pack->writer = td::actor::create_actor<PackageWriter>("writer", new_package);
+
+  for (auto idx = pack->idx + 1; idx < packages_.size(); idx++) {
+    td::unlink(packages_[idx].path).ensure();
+  }
+  packages_.erase(packages_.begin() + pack->idx + 1);
+
+  kv_->commit_transaction().ensure();
+
+  promise.set_value(td::Unit());
+}
+
 }  // namespace validator
 
 }  // namespace ton

validator/db/archive-slice.hpp

@@ -73,7 +73,7 @@ class PackageWriter : public td::actor::Actor {
 
 class ArchiveSlice : public td::actor::Actor {
  public:
-  ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, std::string db_root);
+  ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root);
 
   void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise);
 
@@ -97,6 +97,7 @@ class ArchiveSlice : public td::actor::Actor {
 
   void start_up() override;
   void destroy(td::Promise<td::Unit> promise);
+  void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise);
 
   void begin_transaction();
   void commit_transaction();
@@ -117,6 +118,7 @@ class ArchiveSlice : public td::actor::Actor {
 
   bool key_blocks_only_;
   bool temp_;
+  bool finalized_;
 
   bool destroyed_ = false;
   bool async_mode_ = false;
@@ -130,19 +132,38 @@ class ArchiveSlice : public td::actor::Actor {
 
   struct PackageInfo {
     PackageInfo(std::shared_ptr<Package> package, td::actor::ActorOwn<PackageWriter> writer, BlockSeqno id,
-                std::string path, td::uint32 idx)
-        : package(std::move(package)), writer(std ::move(writer)), id(id), path(std::move(path)), idx(idx) {
+                std::string path, td::uint32 idx, td::uint32 version)
+        : package(std::move(package))
+        , writer(std ::move(writer))
+        , id(id)
+        , path(std::move(path))
+        , idx(idx)
+        , version(version) {
     }
     std::shared_ptr<Package> package;
     td::actor::ActorOwn<PackageWriter> writer;
     BlockSeqno id;
     std::string path;
     td::uint32 idx;
+    td::uint32 version;
   };
   std::vector<PackageInfo> packages_;
 
   td::Result<PackageInfo *> choose_package(BlockSeqno masterchain_seqno, bool force);
-  void add_package(BlockSeqno masterchain_seqno, td::uint64 size);
+  void add_package(BlockSeqno masterchain_seqno, td::uint64 size, td::uint32 version);
+  void truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx, Package *pack);
+  bool truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx, Package *pack);
+
+  void delete_handle(ConstBlockHandle handle);
+  void delete_file(FileReference ref_id);
+  void move_handle(ConstBlockHandle handle, Package *old_pack, Package *pack);
+  void move_file(FileReference ref_id, Package *old_pack, Package *pack);
+
+  BlockSeqno max_masterchain_seqno();
+
+  static constexpr td::uint32 default_package_version() {
+    return 1;
+  }
 };
 
 }  // namespace validator

validator/db/celldb.cpp

@@ -44,9 +44,9 @@ void CellDbIn::start_up() {
   auto empty = get_empty_key_hash();
   if (get_block(empty).is_error()) {
     DbEntry e{get_empty_key(), empty, empty, RootHash::zero()};
-    cell_db_->begin_transaction().ensure();
+    cell_db_->begin_write_batch().ensure();
     set_block(empty, std::move(e));
-    cell_db_->commit_transaction().ensure();
+    cell_db_->commit_write_batch().ensure();
   }
   last_gc_ = empty;
 }
@@ -89,12 +89,12 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promi
   boc_->inc(cell);
   boc_->prepare_commit().ensure();
   vm::CellStorer stor{*cell_db_.get()};
-  cell_db_->begin_transaction().ensure();
+  cell_db_->begin_write_batch().ensure();
   boc_->commit(stor).ensure();
   set_block(empty, std::move(E));
   set_block(D.prev, std::move(P));
   set_block(key_hash, std::move(D));
-  cell_db_->commit_transaction().ensure();
+  cell_db_->commit_write_batch().ensure();
 
   boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot())).ensure();
   td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
@@ -181,12 +181,12 @@ void CellDbIn::gc_cont2(BlockHandle handle) {
   boc_->dec(cell);
   boc_->prepare_commit().ensure();
   vm::CellStorer stor{*cell_db_.get()};
-  cell_db_->begin_transaction().ensure();
+  cell_db_->begin_write_batch().ensure();
   boc_->commit(stor).ensure();
   cell_db_->erase(get_key(last_gc_)).ensure();
   set_block(F.prev, std::move(P));
   set_block(F.next, std::move(N));
-  cell_db_->commit_transaction().ensure();
+  cell_db_->commit_write_batch().ensure();
   alarm_timestamp() = td::Timestamp::now();
 
   boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot())).ensure();

validator/db/rootdb.cpp

@@ -409,10 +409,13 @@ void RootDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::s
   auto merger = StatsMerger::create(std::move(promise));
 }
 
-void RootDb::truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) {
+void RootDb::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise) {
   td::MultiPromise mp;
   auto ig = mp.init_guard();
   ig.add_promise(std::move(promise));
+
+  td::actor::send_closure(archive_db_, &ArchiveManager::truncate, seqno, handle, ig.get_promise());
+  td::actor::send_closure(state_db_, &StateDb::truncate, seqno, handle, ig.get_promise());
 }
 
 void RootDb::add_key_block_proof(td::Ref<Proof> proof, td::Promise<td::Unit> promise) {
@@ -483,8 +486,8 @@ void RootDb::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
   td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise));
 }
 
-void RootDb::run_gc(UnixTime ts) {
-  td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, ts);
+void RootDb::run_gc(UnixTime ts, UnixTime archive_ttl) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, ts, archive_ttl);
 }
 
 }  // namespace validator

validator/db/rootdb.hpp

@@ -34,8 +34,8 @@ namespace validator {
 class RootDb : public Db {
  public:
   enum class Flags : td::uint32 { f_started = 1, f_ready = 2, f_switched = 4, f_archived = 8 };
-  RootDb(td::actor::ActorId<ValidatorManager> validator_manager, std::string root_path, td::uint32 depth)
-      : validator_manager_(validator_manager), root_path_(std::move(root_path)), depth_(depth) {
+  RootDb(td::actor::ActorId<ValidatorManager> validator_manager, std::string root_path)
+      : validator_manager_(validator_manager), root_path_(std::move(root_path)) {
   }
 
   void start_up() override;
@@ -84,7 +84,8 @@ class RootDb : public Db {
   void apply_block(BlockHandle handle, td::Promise<td::Unit> promise) override;
   void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<ConstBlockHandle> promise) override;
   void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<ConstBlockHandle> promise) override;
-  void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<ConstBlockHandle> promise) override;
+  void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno,
+                          td::Promise<ConstBlockHandle> promise) override;
 
   void update_init_masterchain_block(BlockIdExt block, td::Promise<td::Unit> promise) override;
   void get_init_masterchain_block(td::Promise<BlockIdExt> promise) override;
@@ -113,7 +114,7 @@ class RootDb : public Db {
 
   void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) override;
 
-  void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) override;
+  void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise) override;
 
   void add_key_block_proof(td::Ref<Proof> proof, td::Promise<td::Unit> promise) override;
   void add_key_block_proof_link(td::Ref<ProofLink> proof_link, td::Promise<td::Unit> promise) override;
@@ -127,13 +128,12 @@ class RootDb : public Db {
                          td::Promise<td::BufferSlice> promise) override;
   void set_async_mode(bool mode, td::Promise<td::Unit> promise) override;
 
-  void run_gc(UnixTime ts) override;
+  void run_gc(UnixTime ts, UnixTime archive_ttl) override;
 
  private:
   td::actor::ActorId<ValidatorManager> validator_manager_;
 
   std::string root_path_;
-  td::uint32 depth_;
 
   td::actor::ActorOwn<CellDb> cell_db_;
   td::actor::ActorOwn<StateDb> state_db_;

validator/db/statedb.cpp

@@ -29,11 +29,11 @@ namespace validator {
 void StateDb::update_init_masterchain_block(BlockIdExt block, td::Promise<td::Unit> promise) {
   auto key = create_hash_tl_object<ton_api::db_state_key_initBlockId>();
 
-  kv_->begin_transaction().ensure();
+  kv_->begin_write_batch().ensure();
   kv_->set(key.as_slice(),
            create_serialize_tl_object<ton_api::db_state_initBlockId>(create_tl_block_id(block)).as_slice())
       .ensure();
-  kv_->commit_transaction().ensure();
+  kv_->commit_write_batch().ensure();
 
   promise.set_value(td::Unit());
 }
@@ -59,11 +59,11 @@ void StateDb::get_init_masterchain_block(td::Promise<BlockIdExt> promise) {
 void StateDb::update_gc_masterchain_block(BlockIdExt block, td::Promise<td::Unit> promise) {
   auto key = create_hash_tl_object<ton_api::db_state_key_gcBlockId>();
 
-  kv_->begin_transaction().ensure();
+  kv_->begin_write_batch().ensure();
   kv_->set(key.as_slice(),
            create_serialize_tl_object<ton_api::db_state_gcBlockId>(create_tl_block_id(block)).as_slice())
       .ensure();
-  kv_->commit_transaction().ensure();
+  kv_->commit_write_batch().ensure();
 
   promise.set_value(td::Unit());
 }
@@ -89,11 +89,11 @@ void StateDb::get_gc_masterchain_block(td::Promise<BlockIdExt> promise) {
 void StateDb::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise) {
   auto key = create_hash_tl_object<ton_api::db_state_key_shardClient>();
 
-  kv_->begin_transaction().ensure();
+  kv_->begin_write_batch().ensure();
   kv_->set(key.as_slice(),
            create_serialize_tl_object<ton_api::db_state_shardClient>(create_tl_block_id(masterchain_block_id)))
       .ensure();
-  kv_->commit_transaction().ensure();
+  kv_->commit_write_batch().ensure();
 
   promise.set_value(td::Unit());
 }
@@ -120,10 +120,10 @@ void StateDb::update_destroyed_validator_sessions(std::vector<ValidatorSessionId
                                                   td::Promise<td::Unit> promise) {
   auto key = create_hash_tl_object<ton_api::db_state_key_destroyedSessions>();
 
-  kv_->begin_transaction().ensure();
+  kv_->begin_write_batch().ensure();
   kv_->set(key.as_slice(), create_serialize_tl_object<ton_api::db_state_destroyedSessions>(std::move(sessions)))
       .ensure();
-  kv_->commit_transaction().ensure();
+  kv_->commit_write_batch().ensure();
 
   promise.set_value(td::Unit());
 }
@@ -153,9 +153,9 @@ void StateDb::update_async_serializer_state(AsyncSerializerState state, td::Prom
       create_tl_block_id(state.last_block_id), create_tl_block_id(state.last_written_block_id),
       state.last_written_block_ts);
 
-  kv_->begin_transaction().ensure();
+  kv_->begin_write_batch().ensure();
   kv_->set(key.as_slice(), value.as_slice()).ensure();
-  kv_->commit_transaction().ensure();
+  kv_->commit_write_batch().ensure();
 
   promise.set_value(td::Unit());
 }
@@ -188,9 +188,9 @@ void StateDb::update_hardforks(std::vector<BlockIdExt> blocks, td::Promise<td::U
     vec.push_back(create_tl_block_id(e));
   }
 
-  kv_->begin_transaction().ensure();
+  kv_->begin_write_batch().ensure();
   kv_->set(key.as_slice(), create_serialize_tl_object<ton_api::db_state_hardforks>(std::move(vec))).ensure();
-  kv_->commit_transaction();
+  kv_->commit_write_batch();
 
   promise.set_value(td::Unit());
 }
@@ -232,14 +232,95 @@ void StateDb::start_up() {
     auto f = F.move_as_ok();
     CHECK(f->version_ == 2);
   } else {
-    kv_->begin_transaction().ensure();
+    kv_->begin_write_batch().ensure();
     kv_->set(create_serialize_tl_object<ton_api::db_state_key_dbVersion>(),
              create_serialize_tl_object<ton_api::db_state_dbVersion>(2))
         .ensure();
-    kv_->commit_transaction().ensure();
+    kv_->commit_write_batch().ensure();
   }
 }
 
+void StateDb::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise) {
+  {
+    auto key = create_hash_tl_object<ton_api::db_state_key_asyncSerializer>();
+
+    std::string value;
+    auto R = kv_->get(key.as_slice(), value);
+    R.ensure();
+
+    if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+      auto F = fetch_tl_object<ton_api::db_state_asyncSerializer>(value, true);
+      F.ensure();
+      auto obj = F.move_as_ok();
+      if (static_cast<BlockSeqno>(obj->last_->seqno_) > masterchain_seqno) {
+        CHECK(handle);
+        CHECK(handle->inited_unix_time());
+        obj->last_ = create_tl_block_id(handle->id());
+        obj->last_ts_ = handle->unix_time();
+        kv_->begin_write_batch().ensure();
+        kv_->set(key.as_slice(), serialize_tl_object(obj, true)).ensure();
+        kv_->commit_write_batch().ensure();
+      }
+    }
+  }
+  {
+    auto key = create_hash_tl_object<ton_api::db_state_key_shardClient>();
+
+    std::string value;
+    auto R = kv_->get(key.as_slice(), value);
+    R.ensure();
+
+    if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+      auto F = fetch_tl_object<ton_api::db_state_shardClient>(td::BufferSlice{value}, true);
+      F.ensure();
+      auto obj = F.move_as_ok();
+      if (static_cast<BlockSeqno>(obj->block_->seqno_) > masterchain_seqno) {
+        CHECK(handle);
+        obj->block_ = create_tl_block_id(handle->id());
+        kv_->begin_write_batch().ensure();
+        kv_->set(key.as_slice(), serialize_tl_object(obj, true)).ensure();
+        kv_->commit_write_batch().ensure();
+      }
+    }
+  }
+  {
+    auto key = create_hash_tl_object<ton_api::db_state_key_gcBlockId>();
+
+    std::string value;
+    auto R = kv_->get(key.as_slice(), value);
+    R.ensure();
+
+    if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+      auto F = fetch_tl_object<ton_api::db_state_gcBlockId>(td::BufferSlice{value}, true);
+      F.ensure();
+      auto obj = F.move_as_ok();
+      CHECK(static_cast<BlockSeqno>(obj->block_->seqno_) <= masterchain_seqno);
+    }
+  }
+  {
+    auto key = create_hash_tl_object<ton_api::db_state_key_initBlockId>();
+
+    std::string value;
+    auto R = kv_->get(key.as_slice(), value);
+    R.ensure();
+
+    if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+      auto F = fetch_tl_object<ton_api::db_state_initBlockId>(td::BufferSlice{value}, true);
+      F.ensure();
+      auto obj = F.move_as_ok();
+      if (static_cast<BlockSeqno>(obj->block_->seqno_) > masterchain_seqno) {
+        CHECK(handle);
+        obj->block_ = create_tl_block_id(handle->id());
+        kv_->begin_write_batch().ensure();
+        kv_->set(key.as_slice(), serialize_tl_object(obj, true)).ensure();
+        kv_->commit_write_batch().ensure();
+      }
+    }
+  }
+
+  promise.set_value(td::Unit());
+}
+
 }  // namespace validator
 
 }  // namespace ton

validator/db/statedb.hpp

@@ -56,6 +56,7 @@ class StateDb : public td::actor::Actor {
   StateDb(td::actor::ActorId<RootDb> root_db, std::string path);
 
   void start_up() override;
+  void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise);
 
  private:
   using KeyType = td::Bits256;