new db

new database
fift/func bugfixes

CMakeLists.txt

@@ -441,7 +441,7 @@ if (USE_LIBRAPTORQ)
 endif()
 
 add_executable(test-hello-world test/test-hello-world.cpp )
-target_link_libraries(test-hello-world tl_api)
+target_link_libraries(test-hello-world tl_api crypto ton_crypto)
 
 add_executable(test-adnl test/test-adnl.cpp)
 target_link_libraries(test-adnl adnl adnltest dht tl_api)

crypto/CMakeLists.txt

@@ -201,6 +201,7 @@ set(BLOCK_SOURCE
 
 set(SMC_ENVELOPE_SOURCE
   smc-envelope/GenericAccount.cpp
+  smc-envelope/HighloadWallet.cpp
   smc-envelope/MultisigWallet.cpp
   smc-envelope/SmartContract.cpp
   smc-envelope/SmartContractCode.cpp
@@ -210,6 +211,7 @@ set(SMC_ENVELOPE_SOURCE
   smc-envelope/WalletV3.cpp
 
   smc-envelope/GenericAccount.h
+  smc-envelope/HighloadWallet.h
   smc-envelope/MultisigWallet.h
   smc-envelope/SmartContract.h
   smc-envelope/SmartContractCode.h
@@ -387,5 +389,13 @@ if (WINGETOPT_FOUND)
   target_link_libraries_system(dump-block wingetopt)
 endif()
 
+add_executable(adjust-block block/adjust-block.cpp)
+target_include_directories(adjust-block PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
+target_link_libraries(adjust-block PUBLIC ton_crypto fift-lib ton_block)
+if (WINGETOPT_FOUND)
+  target_link_libraries_system(dump-block wingetopt)
+endif()
+
 install(TARGETS fift func RUNTIME DESTINATION bin)
 install(DIRECTORY fift/lib/ DESTINATION lib/fift)

crypto/block/adjust-block.cpp

@@ -0,0 +1,203 @@
+/* 
+    This file is part of TON Blockchain source code.
+
+    TON Blockchain is free software; you can redistribute it and/or
+    modify it under the terms of the GNU General Public License
+    as published by the Free Software Foundation; either version 2
+    of the License, or (at your option) any later version.
+
+    TON Blockchain is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with TON Blockchain.  If not, see <http://www.gnu.org/licenses/>.
+
+    In addition, as a special exception, the copyright holders give permission 
+    to link the code of portions of this program with the OpenSSL library. 
+    You must obey the GNU General Public License in all respects for all 
+    of the code used other than OpenSSL. If you modify file(s) with this 
+    exception, you may extend this exception to your version of the file(s), 
+    but you are not obligated to do so. If you do not wish to do so, delete this 
+    exception statement from your version. If you delete this exception statement 
+    from all source files in the program, then also delete it here.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "block/block.h"
+#include "vm/boc.h"
+#include <iostream>
+#include "block-db.h"
+#include "block-auto.h"
+#include "block-parse.h"
+#include "vm/cp0.h"
+#include "td/utils/crypto.h"
+#include <getopt.h>
+
+using td::Ref;
+using namespace std::literals::string_literals;
+
+int verbosity;
+
+struct IntError {
+  std::string err_msg;
+  IntError(std::string _msg) : err_msg(_msg) {
+  }
+  IntError(const char* _msg) : err_msg(_msg) {
+  }
+};
+
+int fatal(std::string str) {
+  std::cerr << "fatal error: " << str << std::endl;
+  std::exit(2);
+  return 2;
+}
+
+static inline void fail_unless(td::Status res) {
+  if (res.is_error()) {
+    throw IntError{res.to_string()};
+  }
+}
+
+td::Ref<vm::Cell> load_block(std::string filename, ton::BlockIdExt& id) {
+  std::cerr << "loading block from bag-of-cell file " << filename << std::endl;
+  auto bytes_res = block::load_binary_file(filename);
+  if (bytes_res.is_error()) {
+    throw IntError{PSTRING() << "cannot load file `" << filename << "` : " << bytes_res.move_as_error()};
+  }
+  ton::FileHash fhash;
+  td::sha256(bytes_res.ok(), fhash.as_slice());
+  vm::BagOfCells boc;
+  auto res = boc.deserialize(bytes_res.move_as_ok());
+  if (res.is_error()) {
+    throw IntError{PSTRING() << "cannot deserialize bag-of-cells " << res.move_as_error()};
+  }
+  if (res.move_as_ok() <= 0 || boc.get_root_cell().is_null()) {
+    throw IntError{"cannot deserialize bag-of-cells"};
+  }
+  auto root = boc.get_root_cell();
+  std::vector<ton::BlockIdExt> prev;
+  ton::BlockIdExt mc_blkid;
+  bool after_split;
+  fail_unless(block::unpack_block_prev_blk_try(root, id, prev, mc_blkid, after_split, &id));
+  id.file_hash = fhash;
+  std::cerr << "loaded block " << id.to_str() << std::endl;
+  return root;
+}
+
+bool save_block(std::string filename, Ref<vm::Cell> root, ton::BlockIdExt& id) {
+  std::cerr << "saving block into bag-of-cell file " << filename << std::endl;
+  if (root.is_null()) {
+    throw IntError{"new block has no root"};
+  }
+  id.root_hash = root->get_hash().bits();
+  auto res = vm::std_boc_serialize(std::move(root), 31);
+  if (res.is_error()) {
+    throw IntError{PSTRING() << "cannot serialize modified block as a bag-of-cells: "
+                             << res.move_as_error().to_string()};
+  }
+  auto data = res.move_as_ok();
+  td::sha256(data, id.file_hash.as_slice());
+  auto res1 = block::save_binary_file(filename, std::move(data));
+  if (res1.is_error()) {
+    throw IntError{PSTRING() << "cannot save file `" << filename << "` : " << res1};
+  }
+  return true;
+}
+
+Ref<vm::Cell> adjust_block(Ref<vm::Cell> root, int vseqno_incr, const ton::BlockIdExt& id) {
+  std::vector<ton::BlockIdExt> prev;
+  ton::BlockIdExt mc_blkid;
+  bool after_split;
+  fail_unless(block::unpack_block_prev_blk_try(root, id, prev, mc_blkid, after_split));
+  std::cerr << "unpacked header of block " << id.to_str() << std::endl;
+  if (!id.is_masterchain()) {
+    throw IntError{"can modify only masterchain blocks"};
+  }
+  block::gen::Block::Record blk;
+  block::gen::BlockInfo::Record info;
+  if (!(tlb::unpack_cell(root, blk) && tlb::unpack_cell(blk.info, info))) {
+    throw IntError{"cannot unpack block header"};
+  }
+  if (!info.key_block) {
+    throw IntError{"can modify only key blocks"};
+  }
+  info.vert_seqno_incr = true;
+  info.vert_seq_no += vseqno_incr;
+  if (!block::tlb::t_ExtBlkRef.pack_to(info.prev_vert_ref, id, info.end_lt)) {
+    throw IntError{"cannot pack prev_vert_ref"};
+  }
+  if (!(tlb::pack_cell(blk.info, info) && tlb::pack_cell(root, blk))) {
+    throw IntError{"cannot pack block header"};
+  }
+  return root;
+}
+
+void usage() {
+  std::cout << "usage: adjust-block [-i<vs-incr>] <in-boc-file> <out-boc-file>\n\tor adjust-block -h\n\tAdjusts block "
+               "loaded from <in-boc-file> by incrementing vert_seqno by <vs-incr> (1 by default)\n";
+  std::exit(3);
+}
+
+int main(int argc, char* const argv[]) {
+  int i, vseqno_incr = 1;
+  int new_verbosity_level = VERBOSITY_NAME(INFO);
+  std::string in_fname, out_fname;
+  while ((i = getopt(argc, argv, "hi:v:")) != -1) {
+    switch (i) {
+      case 'h':
+        usage();
+        break;
+      case 'i':
+        vseqno_incr = td::to_integer<int>(td::Slice(optarg));
+        CHECK(vseqno_incr > 0 && vseqno_incr < 1000);
+        break;
+      case 'v':
+        new_verbosity_level = VERBOSITY_NAME(FATAL) + (verbosity = td::to_integer<int>(td::Slice(optarg)));
+        break;
+      default:
+        usage();
+        break;
+    }
+  }
+  SET_VERBOSITY_LEVEL(new_verbosity_level);
+  if (argc != optind + 2) {
+    usage();
+    return 2;
+  }
+  in_fname = argv[optind];
+  out_fname = argv[optind + 1];
+  try {
+    ton::BlockIdExt old_id, new_id;
+    auto root = load_block(in_fname, old_id);
+    if (root.is_null()) {
+      return fatal("cannot load BoC from file "s + in_fname);
+    }
+    bool ok = block::gen::t_Block.validate_ref(root);
+    if (!ok) {
+      return fatal("file `"s + in_fname + " does not contain a valid block");
+    }
+    auto adjusted = adjust_block(root, vseqno_incr, old_id);
+    if (adjusted.is_null()) {
+      return fatal("cannot adjust block");
+    }
+    ok = block::gen::t_Block.validate_ref(root);
+    if (!ok) {
+      return fatal("modified block is not valid");
+    }
+    new_id = old_id;
+    if (!save_block(out_fname, adjusted, new_id)) {
+      return fatal("cannot save modified block to file `"s + out_fname + "`");
+    }
+    std::cout << "old block id: " << old_id.to_str() << std::endl;
+    std::cout << "new block id: " << new_id.to_str() << std::endl;
+  } catch (IntError& err) {
+    std::cerr << "internal error: " << err.err_msg << std::endl;
+    return 1;
+  } catch (vm::VmError& err) {
+    std::cerr << "vm error: " << err.get_msg() << std::endl;
+    return 1;
+  }
+  return 0;
+}

crypto/block/block-parse.cpp

@@ -2005,6 +2005,23 @@ bool ExtBlkRef::unpack(Ref<vm::CellSlice> cs_ref, ton::BlockIdExt& blkid, ton::L
   return true;
 }
 
+bool ExtBlkRef::store(vm::CellBuilder& cb, const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const {
+  return cb.store_long_bool(end_lt, 64)            // ext_blk_ref$_ end_lt:uint64
+         && cb.store_long_bool(blkid.seqno(), 32)  // seq_no:uint32
+         && cb.store_bits_bool(blkid.root_hash)    // root_hash:bits256
+         && cb.store_bits_bool(blkid.file_hash);   // file_hash:bits256 = ExtBlkRef;
+}
+
+Ref<vm::Cell> ExtBlkRef::pack_cell(const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const {
+  vm::CellBuilder cb;
+  return store(cb, blkid, end_lt) ? cb.finalize() : Ref<vm::Cell>{};
+}
+
+bool ExtBlkRef::pack_to(Ref<vm::Cell>& cell, const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const {
+  vm::CellBuilder cb;
+  return store(cb, blkid, end_lt) && cb.finalize_to(cell);
+}
+
 const ExtBlkRef t_ExtBlkRef;
 const BlkMasterInfo t_BlkMasterInfo;
 

crypto/block/block-parse.h

@@ -921,6 +921,9 @@ struct ExtBlkRef final : TLB {
   }
   bool unpack(vm::CellSlice& cs, ton::BlockIdExt& blkid, ton::LogicalTime* end_lt = nullptr) const;
   bool unpack(Ref<vm::CellSlice> cs_ref, ton::BlockIdExt& blkid, ton::LogicalTime* end_lt = nullptr) const;
+  bool store(vm::CellBuilder& cb, const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const;
+  Ref<vm::Cell> pack_cell(const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const;
+  bool pack_to(Ref<vm::Cell>& cell, const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const;
 };
 
 extern const ExtBlkRef t_ExtBlkRef;

crypto/block/block.cpp

@@ -1745,7 +1745,7 @@ td::Status unpack_block_prev_blk_ext(Ref<vm::Cell> block_root, const ton::BlockI
   block::gen::ExtBlkRef::Record mcref;  // _ ExtBlkRef = BlkMasterInfo;
   ton::ShardIdFull shard;
   if (!(tlb::unpack_cell(block_root, blk) && tlb::unpack_cell(blk.info, info) && !info.version &&
-        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && !info.vert_seq_no &&
+        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) &&
         (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) {
     return td::Status::Error("cannot unpack block header");
   }
@@ -1809,6 +1809,9 @@ td::Status unpack_block_prev_blk_ext(Ref<vm::Cell> block_root, const ton::BlockI
   } else {
     mc_blkid = ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, mcref.seq_no, mcref.root_hash, mcref.file_hash};
   }
+  if (shard.is_masterchain() && info.vert_seqno_incr && !info.key_block) {
+    return td::Status::Error("non-key masterchain block cannot have vert_seqno_incr set");
+  }
   return td::Status::OK();
 }
 
@@ -1817,7 +1820,7 @@ td::Status check_block_header(Ref<vm::Cell> block_root, const ton::BlockIdExt& i
   block::gen::BlockInfo::Record info;
   ton::ShardIdFull shard;
   if (!(tlb::unpack_cell(block_root, blk) && tlb::unpack_cell(blk.info, info) && !info.version &&
-        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && !info.vert_seq_no)) {
+        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard))) {
     return td::Status::Error("cannot unpack block header");
   }
   ton::BlockId hdr_id{shard, (unsigned)info.seq_no};

crypto/block/dump-block.cpp

@@ -191,8 +191,8 @@ void test2(vm::CellSlice& cs) {
 }
 
 void usage() {
-  std::cout << "usage: test-block [-S][<boc-file>]\n\tor test-block -h\n\tDumps specified blockchain block or state "
-               "from <boc-file>, or runs some tests\n\t-S\tDump a blockchain state\n";
+  std::cout << "usage: dump-block [-S][<boc-file>]\n\tor dump-block -h\n\tDumps specified blockchain block or state "
+               "from <boc-file>, or runs some tests\n\t-S\tDump a blockchain state instead of a block\n";
   std::exit(2);
 }
 

crypto/fift/lib/Color.fif

@@ -0,0 +1,21 @@
+library Color
+{ 27 emit } : esc
+
{ char " word 27 chr swap $+ 1 ' type does create } :_ make-esc"
+
make-esc"[0m" ^reset
+
make-esc"[30m" ^black
+
make-esc"[31m" ^red
+
make-esc"[32m" ^green
+make-esc"[33m" ^yellow
+make-esc"[34m" ^blue
+make-esc"[35m" ^magenta
+make-esc"[36m" ^cyan
+make-esc"[37m" ^white
+
// bold
+make-esc"[30;1m" ^Black
+make-esc"[31;1m" ^Red
+make-esc"[32;1m" ^Green
+make-esc"[33;1m" ^Yellow
+make-esc"[34;1m" ^Blue
+make-esc"[35;1m" ^Magenta
+make-esc"[36;1m" ^Cyan
+make-esc"[37;1m" ^White

crypto/fift/words.cpp

@@ -96,6 +96,10 @@ void interpret_dotstack_list(IntCtx& ctx) {
   *ctx.output_stream << std::endl;
 }
 
+void interpret_dotstack_list_dump(IntCtx& ctx) {
+  ctx.stack.dump(*ctx.output_stream, 3);
+}
+
 void interpret_dump(IntCtx& ctx) {
   ctx.stack.pop_chk().dump(*ctx.output_stream);
   *ctx.output_stream << ' ';
@@ -105,6 +109,10 @@ void interpret_dump_internal(vm::Stack& stack) {
   stack.push_string(stack.pop_chk().to_string());
 }
 
+void interpret_list_dump_internal(vm::Stack& stack) {
+  stack.push_string(stack.pop_chk().to_lisp_string());
+}
+
 void interpret_print_list(IntCtx& ctx) {
   ctx.stack.pop_chk().print_list(*ctx.output_stream);
   *ctx.output_stream << ' ';
@@ -2434,10 +2442,12 @@ void init_words_common(Dictionary& d) {
   d.def_ctx_word("csr. ", interpret_dot_cellslice_rec);
   d.def_ctx_word(".s ", interpret_dotstack);
   d.def_ctx_word(".sl ", interpret_dotstack_list);
+  d.def_ctx_word(".sL ", interpret_dotstack_list_dump);  // TMP
   d.def_ctx_word(".dump ", interpret_dump);
   d.def_ctx_word(".l ", interpret_print_list);
   d.def_ctx_word(".tc ", interpret_dottc);
   d.def_stack_word("(dump) ", interpret_dump_internal);
+  d.def_stack_word("(ldump) ", interpret_list_dump_internal);
   d.def_stack_word("(.) ", interpret_dot_internal);
   d.def_stack_word("(x.) ", std::bind(interpret_dothex_internal, _1, false));
   d.def_stack_word("(X.) ", std::bind(interpret_dothex_internal, _1, true));

crypto/func/builtins.cpp

@@ -949,7 +949,7 @@ void define_builtins() {
   define_builtin_func("~touch2", TypeExpr::new_forall({X, Y}, TypeExpr::new_map(XY, TypeExpr::new_tensor({XY, Unit}))),
                       AsmOp::Nop());
   define_builtin_func("~dump", TypeExpr::new_forall({X}, TypeExpr::new_map(X, TypeExpr::new_tensor({X, Unit}))),
-                      AsmOp::Custom("s0 DUMP", 1, 1));
+                      AsmOp::Custom("s0 DUMP", 1, 1), true);
   define_builtin_func("run_method0", TypeExpr::new_map(Int, Unit),
                       [](auto a, auto b, auto c) { return compile_run_method(a, b, c, 0, false); }, true);
   define_builtin_func("run_method1", TypeExpr::new_forall({X}, TypeExpr::new_map(TypeExpr::new_tensor({Int, X}), Unit)),

crypto/smartcont/elector-code.fc

@@ -792,6 +792,26 @@ _ participant_list() method_id {
   return l;
 }
 
+;; returns the list of all participants of current elections with their data
+_ participant_list_extended() method_id {
+  var elect = get_data().begin_parse().preload_dict();
+  if (elect.null?()) {
+    return nil;
+  }
+  var (elect_at, elect_close, min_stake, total_stake, members, failed, finished) = elect.unpack_elect();
+  var l = nil;
+  var id = (1 << 255) + ((1 << 255) - 1);
+  do {
+    (id, var cs, var f) = members.udict_get_prev?(256, id);
+    if (f) {
+      var (stake, time, max_factor, addr, adnl_addr) = (cs~load_grams(), cs~load_uint(32), cs~load_uint(32), cs~load_uint(256), cs~load_uint(256));
+      cs.end_parse();
+      l = cons(pair(id, tuple4(stake, max_factor, addr, adnl_addr)), l);
+    }
+  } until (~ f);
+  return l;
+}
+
 ;; computes the return stake
 int compute_returned_stake(int wallet_addr) method_id {
   var cs = get_data().begin_parse();

crypto/smartcont/gen-zerostate.fif

@@ -3,9 +3,10 @@
 
 def? $1 { @' $1 } { "" } cond constant suffix 
 { suffix $+ } : +suffix
+256 1<<1- 15 / constant AllOnes
 
 wc_master setworkchain
--17 setglobalid   // negative value means a test instance of the blockchain
+-239 setglobalid   // negative value means a test instance of the blockchain
 
 // Initial state of Workchain 0 (Basic workchain)
 
@@ -54,10 +55,11 @@ Libs{
   x{ABACABADABACABA} s>c public_lib
   x{1234} x{5678} |_ s>c private_lib
 }Libs  // libraries
-GR$1700000000 // balance
+GR$4999990000 // balance
 0 // split_depth
 0 // ticktock
-2 // mode: create
+AllOnes 0 * // address
+6 // mode: create+setaddr
 register_smc
 dup make_special dup constant smc1_addr  
 Masterchain over 
@@ -82,8 +84,8 @@ Masterchain over
 // code
 <b 0 32 u, b> // data
 empty_cell    // libraries
-GR$1000000    // initial balance (1m test Grams)
-0 0 2 register_smc
+GR$1000       // initial balance (1k test Grams)
+0 0 AllOnes 6 * 6 register_smc
 dup make_special dup constant smc2_addr
 Masterchain over 
 2dup ."free test gram giver address = " .addr cr 2dup 6 .Addr cr
@@ -120,7 +122,7 @@ Libs{
   x{ABACABADABACABA} s>c public_lib
   x{1234} x{5678} |_ s>c public_lib
 }Libs  // libraries
-0x333333333 // balance
+GR$666 // balance
 0 // split_depth
 3 // ticktock: tick
 2 // mode: create
@@ -139,7 +141,8 @@ empty_cell  // libraries
 GR$10  // balance: 10 grams
 0 // split_depth
 2 // ticktock: tick
-2 // mode: create
+AllOnes 3 * // address: -1:333...333
+6 // mode: create + setaddr
 register_smc
 dup make_special dup constant smc4_addr dup constant elector_addr
 Masterchain swap
@@ -155,14 +158,15 @@ Masterchain swap
 0 capCreateStats config.version!
 // max-validators max-main-validators min-validators
 // 9 4 1 config.validator_num!
-1000 100 5 config.validator_num!
+1000 100 13 config.validator_num!
 // min-stake max-stake min-total-stake max-factor
 GR$10000 GR$10000000 GR$1000000 sg~10 config.validator_stake_limits!
 // elected-for elect-start-before elect-end-before stakes-frozen-for
 // 400000 200000 4000 400000 config.election_params!
-4000 2000 500 1000 config.election_params!  // DEBUG
+// 4000 2000 500 1000 config.election_params!  // DEBUG
+65536 32768 8192 32768 config.election_params!  // TestNet DEBUG
 // config-addr = -1:5555...5555
-256 1<<1- 3 / constant config_addr
+AllOnes 5 * constant config_addr
 config_addr config.config_smc!
 // elector-addr
 elector_addr config.elector_smc!
@@ -232,8 +236,8 @@ Masterchain swap
  */
  
 // pubkey amount `create-wallet1` or pubkey amount `create-wallet2`
-PK'PuZPPXK5Rff9SvtoS7Y9lUuEixvy-J6aishYFj3Qn6P0pJMb GR$100000000 create-wallet1
-PK'PuYiB1zAWzr4p8j6I681+sGUrRGcn6Ylf7vXl0xaUl/w6Xfg GR$1700000000 create-wallet0
+PK'PuZPPXK5Rff9SvtoS7Y9lUuEixvy-J6aishYFj3Qn6P0pJMb GR$100 create-wallet1
+PK'PuYiB1zAWzr4p8j6I681+sGUrRGcn6Ylf7vXl0xaUl/w6Xfg GR$170 create-wallet0
 
 /*
  *

crypto/smc-envelope/HighloadWallet.cpp

@@ -0,0 +1,126 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "HighloadWallet.h"
+#include "GenericAccount.h"
+#include "SmartContractCode.h"
+
+#include "vm/boc.h"
+#include "vm/cells/CellString.h"
+#include "td/utils/base64.h"
+
+#include <limits>
+
+namespace ton {
+td::Ref<vm::Cell> HighloadWallet::get_init_state(const td::Ed25519::PublicKey& public_key,
+                                                 td::uint32 wallet_id) noexcept {
+  auto code = get_init_code();
+  auto data = get_init_data(public_key, wallet_id);
+  return GenericAccount::get_init_state(std::move(code), std::move(data));
+}
+
+td::Ref<vm::Cell> HighloadWallet::get_init_message(const td::Ed25519::PrivateKey& private_key,
+                                                   td::uint32 wallet_id) noexcept {
+  td::uint32 seqno = 0;
+  td::uint32 valid_until = std::numeric_limits<td::uint32>::max();
+  auto append_message = [&](auto&& cb) -> vm::CellBuilder& {
+    cb.store_long(wallet_id, 32).store_long(valid_until, 32).store_long(seqno, 32);
+    CHECK(cb.store_maybe_ref({}));
+    return cb;
+  };
+  auto signature = private_key.sign(append_message(vm::CellBuilder()).finalize()->get_hash().as_slice()).move_as_ok();
+
+  return append_message(vm::CellBuilder().store_bytes(signature)).finalize();
+}
+
+td::Ref<vm::Cell> HighloadWallet::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id,
+                                                      td::uint32 seqno, td::uint32 valid_until,
+                                                      td::Span<Gift> gifts) noexcept {
+  CHECK(gifts.size() <= 254);
+  vm::Dictionary messages(16);
+  for (size_t i = 0; i < gifts.size(); i++) {
+    auto& gift = gifts[i];
+    td::int32 send_mode = 3;
+    auto gramms = gift.gramms;
+    if (gramms == -1) {
+      gramms = 0;
+      send_mode += 128;
+    }
+    vm::CellBuilder cb;
+    GenericAccount::store_int_message(cb, gift.destination, gramms);
+    cb.store_bytes("\0\0\0\0", 4);
+    //vm::CellString::store(cb, gift.message, 35 * 8).ensure();
+    auto message_inner = cb.finalize();
+    cb = {};
+    cb.store_long(send_mode, 8).store_ref(message_inner);
+    auto key = messages.integer_key(td::make_refint(i), 16, false);
+    messages.set_builder(key.bits(), 16, cb);
+  }
+
+  vm::CellBuilder cb;
+  cb.store_long(wallet_id, 32).store_long(valid_until, 32).store_long(seqno, 32);
+  CHECK(cb.store_maybe_ref(messages.get_root_cell()));
+  auto message_outer = cb.finalize();
+  auto signature = private_key.sign(message_outer->get_hash().as_slice()).move_as_ok();
+  return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize();
+}
+
+td::Ref<vm::Cell> HighloadWallet::get_init_code() noexcept {
+  return SmartContractCode::highload_wallet();
+}
+
+vm::CellHash HighloadWallet::get_init_code_hash() noexcept {
+  return get_init_code()->get_hash();
+}
+
+td::Ref<vm::Cell> HighloadWallet::get_init_data(const td::Ed25519::PublicKey& public_key,
+                                                td::uint32 wallet_id) noexcept {
+  return vm::CellBuilder()
+      .store_long(0, 32)
+      .store_long(wallet_id, 32)
+      .store_bytes(public_key.as_octet_string())
+      .finalize();
+}
+
+td::Result<td::uint32> HighloadWallet::get_seqno() const {
+  return TRY_VM(get_seqno_or_throw());
+}
+
+td::Result<td::uint32> HighloadWallet::get_seqno_or_throw() const {
+  if (state_.data.is_null()) {
+    return 0;
+  }
+  //FIXME use get method
+  return static_cast<td::uint32>(vm::load_cell_slice(state_.data).fetch_ulong(32));
+}
+
+td::Result<td::uint32> HighloadWallet::get_wallet_id() const {
+  return TRY_VM(get_wallet_id_or_throw());
+}
+
+td::Result<td::uint32> HighloadWallet::get_wallet_id_or_throw() const {
+  if (state_.data.is_null()) {
+    return 0;
+  }
+  //FIXME use get method
+  auto cs = vm::load_cell_slice(state_.data);
+  cs.skip_first(32);
+  return static_cast<td::uint32>(cs.fetch_ulong(32));
+}
+
+}  // namespace ton

crypto/smc-envelope/HighloadWallet.h

@@ -0,0 +1,54 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "smc-envelope/SmartContract.h"
+#include "vm/cells.h"
+#include "Ed25519.h"
+#include "block/block.h"
+#include "vm/cells/CellString.h"
+
+namespace ton {
+class HighloadWallet : ton::SmartContract {
+ public:
+  explicit HighloadWallet(State state) : ton::SmartContract(std::move(state)) {
+  }
+  static constexpr unsigned max_message_size = vm::CellString::max_bytes;
+  static td::Ref<vm::Cell> get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) noexcept;
+  static td::Ref<vm::Cell> get_init_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id) noexcept;
+  struct Gift {
+    block::StdAddress destination;
+    td::int64 gramms;
+    std::string message;
+  };
+  static td::Ref<vm::Cell> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id,
+                                               td::uint32 seqno, td::uint32 valid_until, td::Span<Gift> gifts) noexcept;
+
+  static td::Ref<vm::Cell> get_init_code() noexcept;
+  static vm::CellHash get_init_code_hash() noexcept;
+  static td::Ref<vm::Cell> get_init_data(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) noexcept;
+
+  td::Result<td::uint32> get_seqno() const;
+  td::Result<td::uint32> get_wallet_id() const;
+
+ private:
+  td::Result<td::uint32> get_seqno_or_throw() const;
+  td::Result<td::uint32> get_wallet_id_or_throw() const;
+};
+}  // namespace ton

crypto/smc-envelope/SmartContract.cpp

@@ -111,7 +111,7 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref<vm::Stac
   if (GET_VERBOSITY_LEVEL() >= VERBOSITY_NAME(DEBUG)) {
     LOG(DEBUG) << "VM log\n" << logger.res;
     std::ostringstream os;
-    res.stack->dump(os);
+    res.stack->dump(os, 2);
     LOG(DEBUG) << "VM stack:\n" << os.str();
     LOG(DEBUG) << "VM exit code: " << res.code;
     LOG(DEBUG) << "VM accepted: " << res.accepted;

crypto/smc-envelope/SmartContractCode.cpp

@@ -39,6 +39,7 @@ const auto& get_map() {
 #include "smartcont/auto/simple-wallet-ext-code.cpp"
 #include "smartcont/auto/simple-wallet-code.cpp"
 #include "smartcont/auto/wallet-code.cpp"
+#include "smartcont/auto/highload-wallet-code.cpp"
     return map;
   }();
   return map;
@@ -69,4 +70,8 @@ td::Ref<vm::Cell> SmartContractCode::simple_wallet_ext() {
   static auto res = load("simple-wallet-ext").move_as_ok();
   return res;
 }
+td::Ref<vm::Cell> SmartContractCode::highload_wallet() {
+  static auto res = load("highload-wallet").move_as_ok();
+  return res;
+}
 }  // namespace ton

crypto/smc-envelope/SmartContractCode.h

@@ -26,5 +26,6 @@ class SmartContractCode {
   static td::Ref<vm::Cell> wallet();
   static td::Ref<vm::Cell> simple_wallet();
   static td::Ref<vm::Cell> simple_wallet_ext();
+  static td::Ref<vm::Cell> highload_wallet();
 };
 }  // namespace ton

crypto/test/test-smartcont.cpp

@@ -35,6 +35,7 @@
 #include "smc-envelope/TestWallet.h"
 #include "smc-envelope/Wallet.h"
 #include "smc-envelope/WalletV3.h"
+#include "smc-envelope/HighloadWallet.h"
 
 #include "td/utils/base64.h"
 #include "td/utils/crypto.h"
@@ -286,6 +287,73 @@ TEST(Tonlib, WalletV3) {
   CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == gift_message->get_hash());
 }
 
+TEST(Tonlib, HighloadWallet) {
+  auto source_lookup = fift::create_mem_source_lookup(load_source("smartcont/new-highload-wallet.fif")).move_as_ok();
+  source_lookup.write_file("/auto/highload-wallet-code.fif", load_source("smartcont/auto/highload-wallet-code.fif"))
+      .ensure();
+  auto fift_output = fift::mem_run_fift(std::move(source_lookup), {"aba", "0", "239"}).move_as_ok();
+
+  LOG(ERROR) << fift_output.output;
+  auto new_wallet_pk = fift_output.source_lookup.read_file("new-wallet.pk").move_as_ok().data;
+  auto new_wallet_query = fift_output.source_lookup.read_file("new-wallet239-query.boc").move_as_ok().data;
+  auto new_wallet_addr = fift_output.source_lookup.read_file("new-wallet239.addr").move_as_ok().data;
+
+  td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}};
+  auto pub_key = priv_key.get_public_key().move_as_ok();
+  auto init_state = ton::HighloadWallet::get_init_state(pub_key, 239);
+  auto init_message = ton::HighloadWallet::get_init_message(priv_key, 239);
+  auto address = ton::GenericAccount::get_address(0, init_state);
+
+  CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32));
+
+  td::Ref<vm::Cell> res = ton::GenericAccount::create_ext_message(address, init_state, init_message);
+
+  LOG(ERROR) << "---smc-envelope----";
+  vm::load_cell_slice(res).print_rec(std::cerr);
+  LOG(ERROR) << "---fift scripts----";
+  vm::load_cell_slice(vm::std_boc_deserialize(new_wallet_query).move_as_ok()).print_rec(std::cerr);
+  CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == res->get_hash());
+
+  fift_output.source_lookup.write_file("/main.fif", load_source("smartcont/highload-wallet.fif")).ensure();
+  std::string order;
+  std::vector<ton::HighloadWallet::Gift> gifts;
+  auto add_order = [&](td::Slice dest_str, td::int64 gramms) {
+    auto g = td::to_string(gramms);
+    if (g.size() < 10) {
+      g = std::string(10 - g.size(), '0') + g;
+    }
+    order += PSTRING() << "SEND " << dest_str << " " << g.substr(0, g.size() - 9) << "." << g.substr(g.size() - 9)
+                       << "\n";
+
+    ton::HighloadWallet::Gift gift;
+    gift.destination = block::StdAddress::parse(dest_str).move_as_ok();
+    gift.gramms = gramms;
+    gifts.push_back(gift);
+  };
+  std::string dest_str = "Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX";
+  add_order(dest_str, 0);
+  add_order(dest_str, 321000000000ll);
+  add_order(dest_str, 321ll);
+  fift_output.source_lookup.write_file("/order", order).ensure();
+  class ZeroOsTime : public fift::OsTime {
+   public:
+    td::uint32 now() override {
+      return 0;
+    }
+  };
+  fift_output.source_lookup.set_os_time(std::make_unique<ZeroOsTime>());
+  fift_output = fift::mem_run_fift(std::move(fift_output.source_lookup), {"aba", "new-wallet", "239", "123", "order"})
+                    .move_as_ok();
+  auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data;
+  auto gift_message = ton::GenericAccount::create_ext_message(
+      address, {}, ton::HighloadWallet::make_a_gift_message(priv_key, 239, 123, 60, gifts));
+  LOG(ERROR) << "---smc-envelope----";
+  vm::load_cell_slice(gift_message).print_rec(std::cerr);
+  LOG(ERROR) << "---fift scripts----";
+  vm::load_cell_slice(vm::std_boc_deserialize(wallet_query).move_as_ok()).print_rec(std::cerr);
+  CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == gift_message->get_hash());
+}
+
 TEST(Tonlib, TestGiver) {
   auto address =
       block::StdAddress::parse("-1:60c04141c6a7b96d68615e7a91d265ad0f3a9a922e9ae9c901d4fa83f5d3c0d0").move_as_ok();

crypto/vm/continuation.cpp

@@ -692,7 +692,7 @@ int VmState::step() {
   //VM_LOG(st) << "stack:";  stack->dump(VM_LOG(st));
   //VM_LOG(st) << "; cr0.refcnt = " << get_c0()->get_refcnt() - 1 << std::endl;
   if (stack_trace) {
-    stack->dump(std::cerr);
+    stack->dump(std::cerr, 3);
   }
   ++steps;
   if (code->size()) {

crypto/vm/debugops.cpp

@@ -74,7 +74,8 @@ int exec_dump_stack(VmState* st) {
     d = 255;
   }
   for (int i = d; i > 0; i--) {
-    std::cerr << stack[i - 1].to_string() << " ";
+    stack[i - 1].print_list(std::cerr);
+    std::cerr << ' ';
   }
   std::cerr << std::endl;
   return 0;
@@ -85,7 +86,9 @@ int exec_dump_value(VmState* st, unsigned arg) {
   VM_LOG(st) << "execute DUMP s" << arg;
   Stack& stack = st->get_stack();
   if ((int)arg < stack.depth()) {
-    std::cerr << "#DEBUG#: s" << arg << " = " << stack[arg].to_string() << std::endl;
+    std::cerr << "#DEBUG#: s" << arg << " = ";
+    stack[arg].print_list(std::cerr);
+    std::cerr << std::endl;
   } else {
     std::cerr << "#DEBUG#: s" << arg << " is absent" << std::endl;
   }

crypto/vm/dict.cpp

@@ -793,8 +793,7 @@ std::tuple<Ref<CellSlice>, Ref<Cell>, bool> dict_lookup_set(Ref<Cell> dict, td::
 std::pair<Ref<Cell>, bool> pfx_dict_set(Ref<Cell> dict, td::ConstBitPtr key, int m, int n,
                                         const PrefixDictionary::store_value_func_t& store_val,
                                         Dictionary::SetMode mode) {
-  std::cerr << "up to " << n << "-bit prefix code dictionary modification for " << m << "-bit key = " << key.to_hex(m)
-            << std::endl;
+  // std::cerr << "up to " << n << "-bit prefix code dictionary modification for " << m << "-bit key = " << key.to_hex(m) << std::endl;
   if (m > n) {
     return std::make_pair(Ref<Cell>{}, false);
   }

crypto/vm/stack.cpp

@@ -45,6 +45,18 @@ const char* get_exception_msg(Excno exc_no) {
   }
 }
 
+bool StackEntry::is_list(const StackEntry* se) {
+  Ref<Tuple> tuple;
+  while (!se->empty()) {
+    tuple = se->as_tuple_range(2, 2);
+    if (tuple.is_null()) {
+      return false;
+    }
+    se = &tuple->at(1);
+  }
+  return true;
+}
+
 static const char HEX_digits[] = "0123456789ABCDEF";
 
 std::string str_to_hex(std::string data, std::string prefix) {
@@ -62,6 +74,12 @@ std::string StackEntry::to_string() const {
   return std::move(os).str();
 }
 
+std::string StackEntry::to_lisp_string() const {
+  std::ostringstream os;
+  print_list(os);
+  return std::move(os).str();
+}
+
 void StackEntry::dump(std::ostream& os) const {
   switch (tp) {
     case t_null:
@@ -130,6 +148,12 @@ void StackEntry::print_list(std::ostream& os) const {
       break;
     case t_tuple: {
       const auto& tuple = *static_cast<Ref<Tuple>>(ref);
+      if (is_list()) {
+        os << '(';
+        tuple[0].print_list(os);
+        print_list_tail(os, &tuple[1]);
+        break;
+      }
       auto n = tuple.size();
       if (!n) {
         os << "[]";
@@ -137,7 +161,7 @@ void StackEntry::print_list(std::ostream& os) const {
         os << "[";
         tuple[0].print_list(os);
         os << "]";
-      } else if (n != 2) {
+      } else {
         os << "[";
         unsigned c = 0;
         for (const auto& entry : tuple) {
@@ -147,10 +171,6 @@ void StackEntry::print_list(std::ostream& os) const {
           entry.print_list(os);
         }
         os << ']';
-      } else {
-        os << '(';
-        tuple[0].print_list(os);
-        tuple[1].print_list_tail(os);
       }
       break;
     }
@@ -159,26 +179,40 @@ void StackEntry::print_list(std::ostream& os) const {
   }
 }
 
-void StackEntry::print_list_tail(std::ostream& os) const {
-  switch (tp) {
-    case t_null:
-      os << ')';
-      break;
-    case t_tuple: {
-      const auto& tuple = *static_cast<Ref<Tuple>>(ref);
-      if (tuple.size() == 2) {
-        os << ' ';
-        tuple[0].print_list(os);
-        tuple[1].print_list_tail(os);
-        break;
-      }
-    }
-    // fall through
-    default:
+void StackEntry::print_list_tail(std::ostream& os, const StackEntry* se) {
+  Ref<Tuple> tuple;
+  while (!se->empty()) {
+    tuple = se->as_tuple_range(2, 2);
+    if (tuple.is_null()) {
       os << " . ";
-      print_list(os);
-      os << ')';
+      se->print_list(os);
+      break;
+    }
+    os << ' ';
+    tuple->at(0).print_list(os);
+    se = &tuple->at(1);
   }
+  os << ')';
+}
+
+StackEntry StackEntry::make_list(std::vector<StackEntry>&& elems) {
+  StackEntry tail;
+  std::size_t n = elems.size();
+  while (n > 0) {
+    --n;
+    tail = StackEntry{vm::make_tuple_ref(std::move(elems[n]), tail)};
+  }
+  return tail;
+}
+
+StackEntry StackEntry::make_list(const std::vector<StackEntry>& elems) {
+  StackEntry tail;
+  std::size_t n = elems.size();
+  while (n > 0) {
+    --n;
+    tail = StackEntry{vm::make_tuple_ref(elems[n], tail)};
+  }
+  return tail;
 }
 
 StackEntry::StackEntry(Ref<Stack> stack_ref) : ref(std::move(stack_ref)), tp(t_stack) {
@@ -611,13 +645,21 @@ Ref<Stack> Stack::split_top(unsigned top_cnt, unsigned drop_cnt) {
   return new_stk;
 }
 
-void Stack::dump(std::ostream& os, bool cr) const {
+void Stack::dump(std::ostream& os, int mode) const {
   os << " [ ";
-  for (const auto& x : stack) {
-    os << x.to_string() << ' ';
+  if (mode & 2) {
+    for (const auto& x : stack) {
+      x.print_list(os);
+      os << ' ';
+    }
+  } else {
+    for (const auto& x : stack) {
+      x.dump(os);
+      os << ' ';
+    }
   }
   os << "] ";
-  if (cr) {
+  if (mode & 1) {
     os << std::endl;
   }
 }

crypto/vm/stack.hpp

@@ -142,6 +142,12 @@ class StackEntry {
   bool is(int wanted) const {
     return tp == wanted;
   }
+  bool is_list() const {
+    return is_list(this);
+  }
+  static bool is_list(const StackEntry& se) {
+    return is_list(&se);
+  }
   void swap(StackEntry& se) {
     ref.swap(se.ref);
     std::swap(tp, se.tp);
@@ -157,8 +163,9 @@ class StackEntry {
   }
 
  private:
+  static bool is_list(const StackEntry* se);
   template <typename T, Type tag>
-  Ref<T> dynamic_as() const& {
+  Ref<T> dynamic_as() const & {
     return tp == tag ? static_cast<Ref<T>>(ref) : td::Ref<T>{};
   }
   template <typename T, Type tag>
@@ -170,7 +177,7 @@ class StackEntry {
     return tp == tag ? static_cast<Ref<T>>(std::move(ref)) : td::Ref<T>{};
   }
   template <typename T, Type tag>
-  Ref<T> as() const& {
+  Ref<T> as() const & {
     return tp == tag ? Ref<T>{td::static_cast_ref(), ref} : td::Ref<T>{};
   }
   template <typename T, Type tag>
@@ -183,6 +190,8 @@ class StackEntry {
   }
 
  public:
+  static StackEntry make_list(std::vector<StackEntry>&& elems);
+  static StackEntry make_list(const std::vector<StackEntry>& elems);
   template <typename T>
   static StackEntry maybe(Ref<T> ref) {
     if (ref.is_null()) {
@@ -191,31 +200,31 @@ class StackEntry {
       return ref;
     }
   }
-  td::RefInt256 as_int() const& {
+  td::RefInt256 as_int() const & {
     return as<td::CntInt256, t_int>();
   }
   td::RefInt256 as_int() && {
     return move_as<td::CntInt256, t_int>();
   }
-  Ref<Cell> as_cell() const& {
+  Ref<Cell> as_cell() const & {
     return as<Cell, t_cell>();
   }
   Ref<Cell> as_cell() && {
     return move_as<Cell, t_cell>();
   }
-  Ref<CellBuilder> as_builder() const& {
+  Ref<CellBuilder> as_builder() const & {
     return as<CellBuilder, t_builder>();
   }
   Ref<CellBuilder> as_builder() && {
     return move_as<CellBuilder, t_builder>();
   }
-  Ref<CellSlice> as_slice() const& {
+  Ref<CellSlice> as_slice() const & {
     return as<CellSlice, t_slice>();
   }
   Ref<CellSlice> as_slice() && {
     return move_as<CellSlice, t_slice>();
   }
-  Ref<Continuation> as_cont() const&;
+  Ref<Continuation> as_cont() const &;
   Ref<Continuation> as_cont() &&;
   Ref<Cnt<std::string>> as_string_ref() const {
     return as<Cnt<std::string>, t_string>();
@@ -230,16 +239,16 @@ class StackEntry {
   std::string as_bytes() const {
     return tp == t_bytes ? *as_bytes_ref() : "";
   }
-  Ref<Box> as_box() const&;
+  Ref<Box> as_box() const &;
   Ref<Box> as_box() &&;
-  Ref<Tuple> as_tuple() const&;
+  Ref<Tuple> as_tuple() const &;
   Ref<Tuple> as_tuple() &&;
-  Ref<Tuple> as_tuple_range(unsigned max_len = 255, unsigned min_len = 0) const&;
+  Ref<Tuple> as_tuple_range(unsigned max_len = 255, unsigned min_len = 0) const &;
   Ref<Tuple> as_tuple_range(unsigned max_len = 255, unsigned min_len = 0) &&;
-  Ref<Atom> as_atom() const&;
+  Ref<Atom> as_atom() const &;
   Ref<Atom> as_atom() &&;
   template <class T>
-  Ref<T> as_object() const& {
+  Ref<T> as_object() const & {
     return dynamic_as<T, t_object>();
   }
   template <class T>
@@ -248,8 +257,11 @@ class StackEntry {
   }
   void dump(std::ostream& os) const;
   void print_list(std::ostream& os) const;
-  void print_list_tail(std::ostream& os) const;
   std::string to_string() const;
+  std::string to_lisp_string() const;
+
+ private:
+  static void print_list_tail(std::ostream& os, const StackEntry* se);
 };
 
 inline void swap(StackEntry& se1, StackEntry& se2) {
@@ -490,7 +502,8 @@ class Stack : public td::CntObject {
       push(std::move(val));
     }
   }
-  void dump(std::ostream& os, bool cr = true) const;
+  // mode: +1 = add eoln, +2 = Lisp-style lists
+  void dump(std::ostream& os, int mode = 1) const;
 };
 
 }  // namespace vm

lite-client/lite-client-common.cpp

@@ -9,6 +9,7 @@
 using namespace std::literals::string_literals;
 
 namespace liteclient {
+
 td::Result<std::unique_ptr<block::BlockProofChain>> deserialize_proof_chain(
     ton::lite_api::object_ptr<ton::lite_api::liteServer_partialBlockProof> f) {
   // deserialize proof chain
@@ -75,6 +76,7 @@ td::Result<std::unique_ptr<block::BlockProofChain>> deserialize_proof_chain(
   LOG(DEBUG) << "deserialized a BlkProofChain of " << chain->link_count() << " links";
   return std::move(chain);
 }
+
 td::Ref<vm::Tuple> prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, td::Ref<vm::CellSlice> my_addr,
                                  const block::CurrencyCollection& balance) {
   td::BitArray<256> rand_seed;
@@ -96,4 +98,5 @@ td::Ref<vm::Tuple> prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, td::Ref
   LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string();
   return vm::make_tuple_ref(std::move(tuple));
 }
+
 }  // namespace liteclient

lite-client/lite-client-common.h

@@ -5,6 +5,7 @@
 #include "auto/tl/lite_api.hpp"
 
 namespace liteclient {
+
 td::Result<std::unique_ptr<block::BlockProofChain>> deserialize_proof_chain(
     ton::lite_api::object_ptr<ton::lite_api::liteServer_partialBlockProof> f);
 

lite-client/lite-client.cpp

@@ -497,7 +497,7 @@ void TestNode::run_init_queries() {
   get_server_version(0x100);
 }
 
-std::string TestNode::get_word(char delim) {
+td::Slice TestNode::get_word(char delim) {
   if (delim == ' ' || !delim) {
     skipspc();
   }
@@ -506,10 +506,33 @@ std::string TestNode::get_word(char delim) {
     ptr++;
   }
   std::swap(ptr, parse_ptr_);
-  return std::string{ptr, parse_ptr_};
+  return td::Slice{ptr, parse_ptr_};
+}
+
+td::Slice TestNode::get_word_ext(const char* delims, const char* specials) {
+  if (delims[0] == ' ') {
+    skipspc();
+  }
+  const char* ptr = parse_ptr_;
+  while (ptr < parse_end_ && !strchr(delims, *ptr)) {
+    if (specials && strchr(specials, *ptr)) {
+      if (ptr == parse_ptr_) {
+        ptr++;
+      }
+      break;
+    }
+    ptr++;
+  }
+  std::swap(ptr, parse_ptr_);
+  return td::Slice{ptr, parse_ptr_};
 }
 
 bool TestNode::get_word_to(std::string& str, char delim) {
+  str = get_word(delim).str();
+  return !str.empty();
+}
+
+bool TestNode::get_word_to(td::Slice& str, char delim) {
   str = get_word(delim);
   return !str.empty();
 }
@@ -549,12 +572,12 @@ bool TestNode::parse_account_addr(ton::WorkchainId& wc, ton::StdSmcAddress& addr
   return block::parse_std_account_addr(get_word(), wc, addr) || set_error("cannot parse account address");
 }
 
-bool TestNode::convert_uint64(std::string word, td::uint64& val) {
+bool TestNode::convert_uint64(td::Slice word, td::uint64& val) {
   val = ~0ULL;
   if (word.empty()) {
     return false;
   }
-  const char* ptr = word.c_str();
+  const char* ptr = word.data();
   char* end = nullptr;
   val = std::strtoull(ptr, &end, 10);
   if (end == ptr + word.size()) {
@@ -565,12 +588,12 @@ bool TestNode::convert_uint64(std::string word, td::uint64& val) {
   }
 }
 
-bool TestNode::convert_int64(std::string word, td::int64& val) {
+bool TestNode::convert_int64(td::Slice word, td::int64& val) {
   val = (~0ULL << 63);
   if (word.empty()) {
     return false;
   }
-  const char* ptr = word.c_str();
+  const char* ptr = word.data();
   char* end = nullptr;
   val = std::strtoll(ptr, &end, 10);
   if (end == ptr + word.size()) {
@@ -581,7 +604,7 @@ bool TestNode::convert_int64(std::string word, td::int64& val) {
   }
 }
 
-bool TestNode::convert_uint32(std::string word, td::uint32& val) {
+bool TestNode::convert_uint32(td::Slice word, td::uint32& val) {
   td::uint64 tmp;
   if (convert_uint64(word, tmp) && (td::uint32)tmp == tmp) {
     val = (td::uint32)tmp;
@@ -591,7 +614,7 @@ bool TestNode::convert_uint32(std::string word, td::uint32& val) {
   }
 }
 
-bool TestNode::convert_int32(std::string word, td::int32& val) {
+bool TestNode::convert_int32(td::Slice word, td::int32& val) {
   td::int64 tmp;
   if (convert_int64(word, tmp) && (td::int32)tmp == tmp) {
     val = (td::int32)tmp;
@@ -631,6 +654,10 @@ int TestNode::parse_hex_digit(int c) {
   return -1;
 }
 
+bool TestNode::parse_hash(td::Slice str, ton::Bits256& hash) {
+  return str.size() == 64 && parse_hash(str.data(), hash);
+}
+
 bool TestNode::parse_hash(const char* str, ton::Bits256& hash) {
   unsigned char* data = hash.data();
   for (int i = 0; i < 32; i++) {
@@ -690,15 +717,15 @@ bool TestNode::parse_block_id_ext(std::string blkid_str, ton::BlockIdExt& blkid,
 }
 
 bool TestNode::parse_block_id_ext(ton::BlockIdExt& blk, bool allow_incomplete) {
-  return parse_block_id_ext(get_word(), blk, allow_incomplete) || set_error("cannot parse BlockIdExt");
+  return parse_block_id_ext(get_word().str(), blk, allow_incomplete) || set_error("cannot parse BlockIdExt");
 }
 
 bool TestNode::parse_hash(ton::Bits256& hash) {
   auto word = get_word();
-  return (!word.empty() && parse_hash(word.c_str(), hash)) || set_error("cannot parse hash");
+  return parse_hash(word, hash) || set_error("cannot parse hash");
 }
 
-bool TestNode::convert_shard_id(std::string str, ton::ShardIdFull& shard) {
+bool TestNode::convert_shard_id(td::Slice str, ton::ShardIdFull& shard) {
   shard.workchain = ton::workchainInvalid;
   shard.shard = 0;
   auto pos = str.find(':');
@@ -774,7 +801,44 @@ bool TestNode::parse_stack_value(td::Slice str, vm::StackEntry& value) {
 }
 
 bool TestNode::parse_stack_value(vm::StackEntry& value) {
-  return parse_stack_value(td::Slice{get_word()}, value) || set_error("invalid vm stack value");
+  auto word = get_word_ext(" \t", "[()]");
+  if (word.empty()) {
+    return set_error("stack value expected instead of end-of-line");
+  }
+  if (word.size() == 1 && (word[0] == '[' || word[0] == '(')) {
+    int expected = (word[0] == '(' ? ')' : ']');
+    std::vector<vm::StackEntry> values;
+    if (!parse_stack_values(values)) {
+      return false;
+    }
+    word = get_word_ext(" \t", "[()]");
+    if (word.size() != 1 || word[0] != expected) {
+      return set_error("closing bracket expected");
+    }
+    if (expected == ']') {
+      value = vm::StackEntry{std::move(values)};
+    } else {
+      value = vm::StackEntry::make_list(std::move(values));
+    }
+    return true;
+  } else {
+    return parse_stack_value(word, value) || set_error("invalid vm stack value");
+  }
+}
+
+bool TestNode::parse_stack_values(std::vector<vm::StackEntry>& values) {
+  values.clear();
+  while (!seekeoln()) {
+    if (cur() == ']' || cur() == ')') {
+      break;
+    }
+    values.emplace_back();
+    if (!parse_stack_value(values.back())) {
+      values.pop_back();
+      return false;
+    }
+  }
+  return true;
 }
 
 bool TestNode::set_error(std::string err_msg) {
@@ -865,7 +929,7 @@ bool TestNode::do_parse_line() {
   ton::BlockSeqno seqno{};
   ton::UnixTime utime{};
   unsigned count{};
-  std::string word = get_word();
+  std::string word = get_word().str();
   skipspc();
   if (word == "time") {
     return eoln() && get_server_time();
@@ -1027,12 +1091,11 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress
 bool TestNode::parse_run_method(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid,
                                 std::string method_name) {
   std::vector<vm::StackEntry> params;
-  while (!seekeoln()) {
-    vm::StackEntry param;
-    if (!parse_stack_value(param)) {
-      return false;
-    }
-    params.push_back(std::move(param));
+  if (!parse_stack_values(params)) {
+    return set_error("cannot parse list of TVM stack values");
+  }
+  if (!seekeoln()) {
+    return set_error("extra characters after a list of TVM stack values");
   }
   if (!ref_blkid.is_valid()) {
     return set_error("must obtain last block information before making other queries");
@@ -1278,7 +1341,7 @@ void TestNode::run_smc_method(ton::BlockIdExt ref_blk, ton::BlockIdExt blk, ton:
   {
     std::ostringstream os;
     os << "arguments: ";
-    stack->dump(os);
+    stack->dump(os, 3);
     out << os.str();
   }
   long long gas_limit = vm::GasLimits::infty;
@@ -1302,7 +1365,7 @@ void TestNode::run_smc_method(ton::BlockIdExt ref_blk, ton::BlockIdExt blk, ton:
   {
     std::ostringstream os;
     os << "result: ";
-    stack->dump(os);
+    stack->dump(os, 3);
     out << os.str();
   }
 }

lite-client/lite-client.h

@@ -93,6 +93,9 @@ class TestNode : public td::actor::Actor {
   };
 
   void run_init_queries();
+  char cur() const {
+    return *parse_ptr_;
+  }
   bool get_server_time();
   bool get_server_version(int mode = 0);
   void got_server_version(td::Result<td::BufferSlice> res, int mode);
@@ -152,8 +155,10 @@ class TestNode : public td::actor::Actor {
   void got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice res);
   bool do_parse_line();
   bool show_help(std::string command);
-  std::string get_word(char delim = ' ');
+  td::Slice get_word(char delim = ' ');
+  td::Slice get_word_ext(const char* delims, const char* specials = nullptr);
   bool get_word_to(std::string& str, char delim = ' ');
+  bool get_word_to(td::Slice& str, char delim = ' ');
   int skipspc();
   std::string get_line_tail(bool remove_spaces = true) const;
   bool eoln() const;
@@ -164,11 +169,12 @@ class TestNode : public td::actor::Actor {
   bool parse_account_addr(ton::WorkchainId& wc, ton::StdSmcAddress& addr);
   static int parse_hex_digit(int c);
   static bool parse_hash(const char* str, ton::Bits256& hash);
-  static bool convert_uint64(std::string word, td::uint64& val);
-  static bool convert_int64(std::string word, td::int64& val);
-  static bool convert_uint32(std::string word, td::uint32& val);
-  static bool convert_int32(std::string word, td::int32& val);
-  static bool convert_shard_id(std::string str, ton::ShardIdFull& shard);
+  static bool parse_hash(td::Slice str, ton::Bits256& hash);
+  static bool convert_uint64(td::Slice word, td::uint64& val);
+  static bool convert_int64(td::Slice word, td::int64& val);
+  static bool convert_uint32(td::Slice word, td::uint32& val);
+  static bool convert_int32(td::Slice word, td::int32& val);
+  static bool convert_shard_id(td::Slice str, ton::ShardIdFull& shard);
   bool parse_hash(ton::Bits256& hash);
   bool parse_lt(ton::LogicalTime& lt);
   bool parse_uint32(td::uint32& val);
@@ -177,6 +183,7 @@ class TestNode : public td::actor::Actor {
   bool parse_block_id_ext(std::string blk_id_string, ton::BlockIdExt& blkid, bool allow_incomplete = false) const;
   bool parse_stack_value(td::Slice str, vm::StackEntry& value);
   bool parse_stack_value(vm::StackEntry& value);
+  bool parse_stack_values(std::vector<vm::StackEntry>& values);
   bool register_blkid(const ton::BlockIdExt& blkid);
   bool show_new_blkids(bool all = false);
   bool complete_blkid(ton::BlockId partial_blkid, ton::BlockIdExt& complete_blkid) const;

tdutils/CMakeLists.txt

@@ -56,6 +56,7 @@ set(TDUTILS_SOURCE
   td/utils/port/MemoryMapping.cpp
   td/utils/port/path.cpp
   td/utils/port/PollFlags.cpp
+  td/utils/port/rlimit.cpp
   td/utils/port/ServerSocketFd.cpp
   td/utils/port/signals.cpp
   td/utils/port/sleep.cpp
@@ -129,6 +130,7 @@ set(TDUTILS_SOURCE
   td/utils/port/Poll.h
   td/utils/port/PollBase.h
   td/utils/port/PollFlags.h
+  td/utils/port/rlimit.h
   td/utils/port/RwMutex.h
   td/utils/port/ServerSocketFd.h
   td/utils/port/signals.h

tdutils/td/utils/misc.h

@@ -301,6 +301,20 @@ typename std::enable_if<std::is_unsigned<T>::value, T>::type hex_to_integer(Slic
   return integer_value;
 }
 
+template <class T>
+Result<typename std::enable_if<std::is_unsigned<T>::value, T>::type> hex_to_integer_safe(Slice str) {
+  T integer_value = 0;
+  auto begin = str.begin();
+  auto end = str.end();
+  while (begin != end) {
+    if (!is_hex_digit(*begin)) {
+      return Status::Error("not a hex digit");
+    }
+    integer_value = static_cast<T>(integer_value * 16 + hex_to_int(*begin++));
+  }
+  return integer_value;
+}
+
 double to_double(Slice str);
 
 template <class T>

tdutils/td/utils/port/rlimit.cpp

@@ -0,0 +1,84 @@
+#include "rlimit.h"
+#if TD_LINUX
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#endif
+
+namespace td {
+
+#if TD_PORT_POSIX
+
+namespace {
+
+int get_rlimit_type(RlimitType rlim_type) {
+  switch (rlim_type) {
+    case RlimitType::nofile:
+      return RLIMIT_NOFILE;
+    case RlimitType::rss:
+      return RLIMIT_RSS;
+    default:
+      UNREACHABLE();
+  }
+}
+
+}  // namespace
+
+td::Status change_rlimit(RlimitType rlim_type, td::uint64 value, td::uint64 cap) {
+  if (cap && value > cap) {
+    return td::Status::Error("setrlimit(): bad argument");
+  }
+  int resource = get_rlimit_type(rlim_type);
+
+  struct rlimit r;
+  if (getrlimit(resource, &r) < 0) {
+    return td::Status::PosixError(errno, "failed getrlimit()");
+  }
+
+  if (cap) {
+    r.rlim_max = cap;
+  } else if (r.rlim_max < value) {
+    r.rlim_max = value;
+  }
+  r.rlim_cur = value;
+  if (setrlimit(resource, &r) < 0) {
+    return td::Status::PosixError(errno, "failed setrlimit()");
+  }
+  return td::Status::OK();
+}
+
+td::Status change_maximize_rlimit(RlimitType rlim_type, td::uint64 value) {
+  int resource = get_rlimit_type(rlim_type);
+
+  struct rlimit r;
+  if (getrlimit(resource, &r) < 0) {
+    return td::Status::PosixError(errno, "failed getrlimit()");
+  }
+
+  if (r.rlim_max < value) {
+    auto t = r;
+    t.rlim_cur = value;
+    t.rlim_max = value;
+    if (setrlimit(resource, &t) >= 0) {
+      return td::Status::OK();
+    }
+  }
+
+  r.rlim_cur = value < r.rlim_max ? value : r.rlim_max;
+  if (setrlimit(resource, &r) < 0) {
+    return td::Status::PosixError(errno, "failed setrlimit()");
+  }
+  return td::Status::OK();
+}
+#else
+td::Status change_rlimit(RlimitType rlim, td::uint64 value) {
+  return td::Status::Error("setrlimit not implemented on WINDOWS");
+}
+td::Status change_maximize_rlimit(RlimitType rlim, td::uint64 value) {
+  return td::Status::OK();
+}
+#endif
+
+}  // namespace td
+

tdutils/td/utils/port/rlimit.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "td/utils/port/config.h"
+#include "td/utils/port/platform.h"
+#include "td/utils/Status.h"
+
+namespace td {
+
+enum class RlimitType { nofile, rss };
+
+td::Status change_rlimit(RlimitType rlim_type, td::uint64 value, td::uint64 cap = 0);
+td::Status change_maximize_rlimit(RlimitType rlim, td::uint64 value);
+
+}  // namespace td

test/regression-tests.ans

@@ -7,7 +7,7 @@ Test_Fift_bug_newlize_default e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca4
 Test_Fift_bug_ufits_default 51bf5a9f1ed7633a193f6fdd17a7a3af8e032dfe72a9669c85e8639aa8a7c195
 Test_Fift_contfrac_default 09ebce5c91bcb70696c6fb6981d82dc3b9e3444dab608a7a1b044c0ddd778a96
 Test_Fift_test_default 4e44b3382963ec89f7b5c8f2ebd85da3bc8aebad5b49f5b11b14075061477b4d
-Test_Fift_test_dict_default 480d22a6ec25a232febf4eec8ff64747573f79721327e7ff3b1aa7ea4921bbb4
+Test_Fift_test_dict_default 1879f03e5fb25dcdd33f40120a6d352c246481895c9f8e45975bf3e101d9ee70
 Test_Fift_test_fixed_default 278a19d56b773102caf5c1fe2997ea6c8d0d9e720eff8503feede6398a197eec
 Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a
 Test_Fift_test_sort_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a

test/test-hello-world.cpp

@@ -31,6 +31,7 @@
 #include "auto/tl/ton_api_json.h"
 
 #include "tl/tl_json.h"
+#include "td/utils/Random.h"
 
 namespace {
 std::string config = R"json(
@@ -135,4 +136,25 @@ int main() {
     return res;
   };
   test_tl_json(ton::ton_api::make_object<ton::ton_api::testVectorBytes>(create_vector_bytes()));
+
+  td::Bits256 x;
+  td::Random::secure_bytes(x.as_slice());
+
+  auto s = x.to_hex();
+
+  auto v = td::hex_decode(s).move_as_ok();
+
+  auto w = td::buffer_to_hex(x.as_slice());
+
+  td::Bits256 y;
+  y.as_slice().copy_from(v);
+
+  CHECK(x == y);
+
+  auto w2 = td::hex_decode(w).move_as_ok();
+  td::Bits256 z;
+  z.as_slice().copy_from(w2);
+
+  LOG_CHECK(x == z) << s << " " << w;
+  return 0;
 }

test/test-ton-collator.cpp

@@ -303,6 +303,10 @@ class TestNode : public td::actor::Actor {
       void get_next_key_blocks(ton::BlockIdExt block_id, td::Timestamp timeout,
                                td::Promise<std::vector<ton::BlockIdExt>> promise) override {
       }
+      void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+
+                            td::Promise<std::string> promise) override {
+      }
 
       void new_key_block(ton::validator::BlockHandle handle) override {
       }

tl/generate/scheme/ton_api.tl

@@ -362,6 +362,9 @@ tonNode.capabilities version:int capabilities:long = tonNode.Capabilities;
 
 tonNode.success = tonNode.Success;
 
+tonNode.archiveNotFound = tonNode.ArchiveInfo;
+tonNode.archiveInfo id:long = tonNode.ArchiveInfo;
+
 ---functions---
 
 tonNode.getNextBlockDescription prev_block:tonNode.blockIdExt = tonNode.BlockDescription;
@@ -385,6 +388,8 @@ tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data;
 tonNode.downloadBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
 tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data;
 tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
+tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo;
+tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data;
 
 tonNode.getCapabilities = tonNode.Capabilities;
 
@@ -417,7 +422,8 @@ db.block.info#4ac6e727 id:tonNode.blockIdExt flags:# prev_left:flags.1?tonNode.b
                                             next_right:flags.4?tonNode.blockIdExt
                                             lt:flags.13?long 
                                             ts:flags.14?int
-                                            state:flags.17?int256 = db.block.Info;
+                                            state:flags.17?int256 
+                                            masterchain_ref_seqno:flags.23?int = db.block.Info;
 db.block.packedInfo id:tonNode.blockIdExt unixtime:int offset:long = db.block.Info;
 db.block.archivedInfo id:tonNode.blockIdExt flags:# next:flags.0?tonNode.blockIdExt = db.block.Info;
 
@@ -447,6 +453,7 @@ db.state.gcBlockId block:tonNode.blockIdExt = db.state.GcBlockId;
 db.state.shardClient block:tonNode.blockIdExt = db.state.ShardClient;
 db.state.asyncSerializer block:tonNode.blockIdExt last:tonNode.blockIdExt last_ts:int = db.state.AsyncSerializer;
 db.state.hardforks blocks:(vector tonNode.blockIdExt) = db.state.Hardforks;
+db.state.dbVersion version:int = db.state.DbVersion;
 
 db.state.key.destroyedSessions = db.state.Key;
 db.state.key.initBlockId = db.state.Key;
@@ -454,6 +461,7 @@ db.state.key.gcBlockId = db.state.Key;
 db.state.key.shardClient = db.state.Key;
 db.state.key.asyncSerializer = db.state.Key;
 db.state.key.hardforks = db.state.Key;
+db.state.key.dbVersion = db.state.Key;
 
 db.lt.el.key workchain:int shard:long idx:int = db.lt.Key;
 db.lt.desc.key workchain:int shard:long = db.lt.Key;
@@ -464,12 +472,13 @@ db.lt.desc.value first_idx:int last_idx:int last_seqno:int last_lt:long last_ts:
 db.lt.shard.value workchain:int shard:long = db.lt.shard.Value; 
 db.lt.status.value total_shards:int = db.lt.status.Value;
 
-db.archive.index.key = db.archive.Key;
-db.archive.package.key unixtime:int key:Bool = db.archive.Key;
+db.files.index.key = db.files.Key;
+db.files.package.key package_id:int key:Bool temp:Bool = db.files.Key;
 
-db.archive.index.value packages:(vector int) key_packages:(vector int) = db.archive.index.Value;
-db.archive.package.firstBlock workchain:int shard:long seqno:int lt:long = db.archive.package.FirstBlock;
-db.archive.package.value unixtime:int key:Bool firstblocks:(vector db.archive.package.firstBlock) deleted:Bool = db.archive.package.Value;
+db.files.index.value packages:(vector int) key_packages:(vector int) temp_packages:(vector int) = db.files.index.Value;
+db.files.package.firstBlock workchain:int shard:long seqno:int unixtime:int lt:long = db.files.package.FirstBlock;
+db.files.package.value package_id:int key:Bool temp:Bool firstblocks:(vector db.files.package.firstBlock) deleted:Bool 
+                   = db.files.package.Value; 
 
 ---functions---
 

tl/generate/scheme/tonlib_api.tl

@@ -23,6 +23,7 @@ config config:string blockchain_name:string use_callbacks_for_network:Bool ignor
 
 options config:config keystore_type:KeyStoreType = Options;
 options.configInfo default_wallet_id:int53 = options.ConfigInfo;
+options.info config_info:options.configInfo = options.Info;
 
 key public_key:string secret:secureBytes = Key;
 inputKeyRegular key:key local_password:secureBytes = InputKey;
@@ -81,10 +82,14 @@ query.info id:int53 valid_until:int53 body_hash:bytes  = query.Info;
 tvm.slice bytes:string = tvm.Slice;
 tvm.cell bytes:string = tvm.Cell;
 tvm.numberDecimal number:string = tvm.Number;
+tvm.tuple elements:vector<tvm.StackEntry> = tvm.Tuple;
+tvm.list elements:vector<tvm.StackEntry> = tvm.List;
 
 tvm.stackEntrySlice slice:tvm.slice = tvm.StackEntry;
 tvm.stackEntryCell cell:tvm.cell = tvm.StackEntry;
 tvm.stackEntryNumber number:tvm.Number = tvm.StackEntry;
+tvm.stackEntryTuple tuple:tvm.Tuple = tvm.StackEntry;
+tvm.stackEntryList list:tvm.List = tvm.StackEntry;
 tvm.stackEntryUnsupported = tvm.StackEntry;
 
 smc.info id:int53 = smc.Info;
@@ -121,10 +126,10 @@ liteServer.info now:int53 version:int32 capabilities:int64 = liteServer.Info;
 
 ---functions---
 
-init options:options = Ok;
+init options:options = options.Info;
 close = Ok;
 
-options.setConfig config:config = Ok;
+options.setConfig config:config = options.ConfigInfo;
 options.validateConfig config:config = options.ConfigInfo;
 
 createNewKey local_password:secureBytes mnemonic_password:secureBytes random_extra_seed:secureBytes = Key;

ton/ton-types.h

@@ -23,6 +23,7 @@
 #include "td/utils/bits.h"
 #include "td/utils/Slice.h"
 #include "td/utils/UInt.h"
+#include "td/utils/misc.h"
 
 namespace ton {
 
@@ -158,6 +159,9 @@ struct BlockId {
   explicit operator ShardIdFull() const {
     return ShardIdFull{workchain, shard};
   }
+  ShardIdFull shard_full() const {
+    return ShardIdFull{workchain, shard};
+  }
   bool is_valid() const {
     return workchain != workchainInvalid;
   }
@@ -274,6 +278,23 @@ struct BlockIdExt {
   std::string to_str() const {
     return id.to_str() + ':' + root_hash.to_hex() + ':' + file_hash.to_hex();
   }
+  static td::Result<BlockIdExt> from_str(td::CSlice s) {
+    BlockIdExt v;
+    char rh[65];
+    char fh[65];
+    auto r = sscanf(s.begin(), "(%d,%lx,%u):%64s:%64s", &v.id.workchain, &v.id.shard, &v.id.seqno, rh, fh);
+    if (r < 5) {
+      return td::Status::Error("failed to parse block id");
+    }
+    if (strlen(rh) != 64 || strlen(fh) != 64) {
+      return td::Status::Error("failed to parse block id: bad roothash/filehash");
+    }
+    TRY_RESULT(re, td::hex_decode(td::Slice(rh, 64)));
+    v.root_hash.as_slice().copy_from(td::Slice(re));
+    TRY_RESULT(fe, td::hex_decode(td::Slice(fh, 64)));
+    v.file_hash.as_slice().copy_from(td::Slice(fe));
+    return v;
+  }
 };
 
 struct ZeroStateIdExt {

tonlib/test/offline.cpp

@@ -438,3 +438,94 @@ TEST(Tonlib, KeysApi) {
   CHECK(new_imported_key->public_key_ == key->public_key_);
   CHECK(new_imported_key->secret_ != key->secret_);
 }
+
+TEST(Tonlib, ConfigCache) {
+  using tonlib_api::make_object;
+  Client client;
+
+  td::mkdir("testdir").ignore();
+  // init
+  sync_send(client, make_object<tonlib_api::init>(make_object<tonlib_api::options>(
+                        nullptr, make_object<tonlib_api::keyStoreTypeDirectory>("testdir"))))
+      .ensure();
+
+  auto testnet = R"abc({
+  "liteservers": [
+  ],
+  "validator": {
+    "@type": "validator.config.global",
+    "zero_state": {
+      "workchain": -1,
+      "shard": -9223372036854775808,
+      "seqno": 0,
+      "root_hash": "VCSXxDHhTALFxReyTZRd8E4Ya3ySOmpOWAS4rBX9XBY=",
+      "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo="
+    }
+  }
+})abc";
+  auto testnet2 = R"abc({
+  "liteservers": [
+  ],
+  "validator": {
+    "@type": "validator.config.global",
+    "zero_state": {
+      "workchain": -1,
+      "shard": -9223372036854775808,
+      "seqno": 0,
+      "root_hash": "VXSXxDHhTALFxReyTZRd8E4Ya3ySOmpOWAS4rBX9XBY=",
+      "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo="
+    }
+  }
+})abc";
+  auto testnet3 = R"abc({
+  "liteservers": [
+  ],
+  "validator": {
+    "@type": "validator.config.global",
+    "zero_state": {
+      "workchain": -1,
+      "shard": -9223372036854775808,
+      "seqno": 0,
+      "root_hash": "ZXSXxDHhTALFxReyTZRd8E4Ya3ySOmpOWAS4rBX9XBY=",
+      "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo="
+    }
+  }
+})abc";
+  auto bad = R"abc({
+  "liteservers": [
+  ],
+  "validator": {
+    "@type": "validator.config.global",
+    "zero_state": {
+      "workchain": -1,
+      "shard": -9223372036854775808,
+      "seqno": 0,
+      "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo="
+    }
+  }
+})abc";
+  sync_send(client,
+            make_object<tonlib_api::options_validateConfig>(make_object<tonlib_api::config>(bad, "", true, false)))
+      .ensure_error();
+
+  sync_send(client,
+            make_object<tonlib_api::options_validateConfig>(make_object<tonlib_api::config>(testnet, "", true, false)))
+      .ensure();
+  sync_send(client,
+            make_object<tonlib_api::options_validateConfig>(make_object<tonlib_api::config>(testnet2, "", true, false)))
+      .ensure();
+  sync_send(client,
+            make_object<tonlib_api::options_validateConfig>(make_object<tonlib_api::config>(testnet3, "", true, false)))
+      .ensure();
+
+  sync_send(client, make_object<tonlib_api::options_validateConfig>(
+                        make_object<tonlib_api::config>(testnet2, "testnet", true, false)))
+      .ensure_error();
+
+  sync_send(client, make_object<tonlib_api::options_setConfig>(
+                        make_object<tonlib_api::config>(testnet2, "testnet2", true, false)))
+      .ensure();
+  sync_send(client, make_object<tonlib_api::options_setConfig>(
+                        make_object<tonlib_api::config>(testnet3, "testnet2", true, false)))
+      .ensure_error();
+}

tonlib/test/online.cpp

@@ -533,14 +533,12 @@ int main(int argc, char* argv[]) {
 
   Client client;
   {
-    auto info = sync_send(client, make_object<tonlib_api::options_validateConfig>(
-                                      make_object<tonlib_api::config>(global_config_str, "", false, false)))
+    auto info = sync_send(client, make_object<tonlib_api::init>(make_object<tonlib_api::options>(
+                                      make_object<tonlib_api::config>(global_config_str, "", false, false),
+                                      make_object<tonlib_api::keyStoreTypeDirectory>(keystore_dir))))
                     .move_as_ok();
-    default_wallet_id = static_cast<td::uint32>(info->default_wallet_id_);
-    sync_send(client, make_object<tonlib_api::init>(make_object<tonlib_api::options>(
-                          make_object<tonlib_api::config>(global_config_str, "", false, false),
-                          make_object<tonlib_api::keyStoreTypeDirectory>(keystore_dir))))
-        .ensure();
+    default_wallet_id = static_cast<td::uint32>(info->config_info_->default_wallet_id_);
+    LOG(ERROR) << default_wallet_id;
   }
 
   // wait till client is synchronized with blockchain.

tonlib/tonlib/TonlibClient.cpp

@@ -66,6 +66,16 @@ struct SendMessage {
 };
 }  // namespace int_api
 
+template <class R, class O, class F>
+R downcast_call2(O&& o, F&& f, R res = {}) {
+  downcast_call(o, [&](auto& x) { res = f(x); });
+  return res;
+}
+
+tonlib_api::object_ptr<tonlib_api::options_configInfo> to_tonlib_api(const TonlibClient::FullConfig& full_config) {
+  return tonlib_api::make_object<tonlib_api::options_configInfo>(full_config.wallet_id);
+}
+
 class TonlibQueryActor : public td::actor::Actor {
  public:
   TonlibQueryActor(td::actor::ActorShared<TonlibClient> client) : client_(std::move(client)) {
@@ -867,7 +877,7 @@ void TonlibClient::update_last_block_state(LastBlockState state, td::uint32 conf
     return;
   }
 
-  last_block_storage_.save_state(blockchain_name_, state);
+  last_block_storage_.save_state(last_state_key_, state);
 }
 
 void TonlibClient::update_sync_state(LastBlockSyncState state, td::uint32 config_generation) {
@@ -889,7 +899,7 @@ void TonlibClient::update_sync_state(LastBlockSyncState state, td::uint32 config
   }
 }
 
-void TonlibClient::init_last_block(td::optional<Config> o_master_config) {
+void TonlibClient::init_last_block(LastBlockState state) {
   ref_cnt_++;
   class Callback : public LastBlock::Callback {
    public:
@@ -907,31 +917,8 @@ void TonlibClient::init_last_block(td::optional<Config> o_master_config) {
     td::actor::ActorShared<TonlibClient> client_;
     td::uint32 config_generation_;
   };
-  LastBlockState state;
 
-  td::Result<LastBlockState> r_state;
-  if (!ignore_cache_) {
-    r_state = last_block_storage_.get_state(blockchain_name_);
-  }
-  if (ignore_cache_ || r_state.is_error()) {
-    LOG_IF(WARNING, !ignore_cache_) << "Unknown LastBlockState: " << r_state.error();
-    state.zero_state_id = ton::ZeroStateIdExt(config_.zero_state_id.id.workchain, config_.zero_state_id.root_hash,
-                                              config_.zero_state_id.file_hash),
-    state.last_block_id = config_.zero_state_id;
-    state.last_key_block_id = config_.zero_state_id;
-    last_block_storage_.save_state(blockchain_name_, state);
-  } else {
-    state = r_state.move_as_ok();
-  }
-
-  if (o_master_config) {
-    auto master_config = o_master_config.unwrap();
-    if (master_config.init_block_id.is_valid() &&
-        state.last_key_block_id.id.seqno < master_config.init_block_id.id.seqno) {
-      state.last_key_block_id = master_config.init_block_id;
-      LOG(INFO) << "Use init block from MASTER config: " << master_config.init_block_id.to_str();
-    }
-  }
+  last_block_storage_.save_state(last_state_key_, state);
 
   raw_last_block_ = td::actor::create_actor<LastBlock>(
       td::actor::ActorOptions().with_name("LastBlock").with_poll(false), get_client_ref(), std::move(state), config_,
@@ -1013,8 +1000,8 @@ tonlib_api::object_ptr<tonlib_api::Object> TonlibClient::static_request(
     return tonlib_api::make_object<tonlib_api::error>(400, "Request is empty");
   }
 
-  tonlib_api::object_ptr<tonlib_api::Object> response;
-  downcast_call(*function, [&response](auto& request) { response = TonlibClient::do_static_request(request); });
+  auto response = downcast_call2<tonlib_api::object_ptr<tonlib_api::Object>>(
+      *function, [](auto& request) { return TonlibClient::do_static_request(request); });
   VLOG(tonlib_query) << "  answer static query " << to_string(response);
   return response;
 }
@@ -1029,7 +1016,6 @@ bool TonlibClient::is_static_request(td::int32 id) {
     case tonlib_api::testGiver_getAccountAddress::ID:
     case tonlib_api::packAccountAddress::ID:
     case tonlib_api::unpackAccountAddress::ID:
-    case tonlib_api::options_validateConfig::ID:
     case tonlib_api::getBip39Hints::ID:
     case tonlib_api::setLogStream::ID:
     case tonlib_api::getLogStream::ID:
@@ -1190,7 +1176,7 @@ tonlib_api::object_ptr<tonlib_api::Object> TonlibClient::do_static_request(tonli
 }
 
 td::Status TonlibClient::do_request(const tonlib_api::init& request,
-                                    td::Promise<object_ptr<tonlib_api::ok>>&& promise) {
+                                    td::Promise<object_ptr<tonlib_api::options_info>>&& promise) {
   if (state_ != State::Uninited) {
     return td::Status::Error(400, "Tonlib is already inited");
   }
@@ -1201,23 +1187,24 @@ td::Status TonlibClient::do_request(const tonlib_api::init& request,
     return TonlibError::EmptyField("options.keystore_type");
   }
 
-  td::Result<td::unique_ptr<KeyValue>> r_kv;
-  downcast_call(
+  auto r_kv = downcast_call2<td::Result<td::unique_ptr<KeyValue>>>(
       *request.options_->keystore_type_,
       td::overloaded(
-          [&](tonlib_api::keyStoreTypeDirectory& directory) { r_kv = KeyValue::create_dir(directory.directory_); },
-          [&](tonlib_api::keyStoreTypeInMemory& inmemory) { r_kv = KeyValue::create_inmemory(); }));
+          [](tonlib_api::keyStoreTypeDirectory& directory) { return KeyValue::create_dir(directory.directory_); },
+          [](tonlib_api::keyStoreTypeInMemory& inmemory) { return KeyValue::create_inmemory(); }));
   TRY_RESULT(kv, std::move(r_kv));
   kv_ = std::shared_ptr<KeyValue>(kv.release());
 
   key_storage_.set_key_value(kv_);
   last_block_storage_.set_key_value(kv_);
+  auto res = tonlib_api::make_object<tonlib_api::options_info>();
   if (request.options_->config_) {
     TRY_RESULT(full_config, validate_config(std::move(request.options_->config_)));
+    res->config_info_ = to_tonlib_api(full_config);
     set_config(std::move(full_config));
   }
   state_ = State::Running;
-  promise.set_value(tonlib_api::make_object<tonlib_api::ok>());
+  promise.set_value(std::move(res));
   return td::Status::OK();
 }
 
@@ -1291,21 +1278,55 @@ td::Result<TonlibClient::FullConfig> TonlibClient::validate_config(tonlib_api::o
   }
 
   td::optional<Config> o_master_config;
+  std::string last_state_key;
   if (config->blockchain_name_.empty()) {
+    last_state_key = new_config.zero_state_id.root_hash.as_slice().str();
     o_master_config = get_default_master_config().by_root_hash(new_config.zero_state_id.root_hash);
   } else {
+    last_state_key = config->blockchain_name_;
     o_master_config = get_default_master_config().by_name(config->blockchain_name_);
   }
 
   if (o_master_config && o_master_config.value().zero_state_id != new_config.zero_state_id) {
     return TonlibError::InvalidConfig("zero_state differs from embedded zero_state");
   }
+
+  LastBlockState state;
+  td::Result<LastBlockState> r_state;
+  if (!config->ignore_cache_) {
+    r_state = last_block_storage_.get_state(last_state_key);
+  }
+  auto zero_state = ton::ZeroStateIdExt(new_config.zero_state_id.id.workchain, new_config.zero_state_id.root_hash,
+                                        new_config.zero_state_id.file_hash);
+  if (config->ignore_cache_ || r_state.is_error()) {
+    LOG_IF(WARNING, !config->ignore_cache_) << "Unknown LastBlockState: " << r_state.error();
+    state.zero_state_id = zero_state;
+    state.last_block_id = new_config.zero_state_id;
+    state.last_key_block_id = new_config.zero_state_id;
+  } else {
+    state = r_state.move_as_ok();
+    if (state.zero_state_id != zero_state) {
+      LOG(ERROR) << state.zero_state_id.to_str() << " " << zero_state.to_str();
+      return TonlibError::InvalidConfig("zero_state differs from cached zero_state");
+    }
+  }
+
+  if (o_master_config) {
+    auto master_config = o_master_config.unwrap();
+    if (master_config.init_block_id.is_valid() &&
+        state.last_key_block_id.id.seqno < master_config.init_block_id.id.seqno) {
+      state.last_key_block_id = master_config.init_block_id;
+      LOG(INFO) << "Use init block from MASTER config: " << master_config.init_block_id.to_str();
+    }
+  }
+
   FullConfig res;
   res.config = std::move(new_config);
-  res.o_master_config = std::move(o_master_config);
-  res.ignore_cache = config->ignore_cache_;
   res.use_callbacks_for_network = config->use_callbacks_for_network_;
   res.wallet_id = td::as<td::uint32>(res.config.zero_state_id.root_hash.as_slice().data());
+  res.last_state_key = std::move(last_state_key);
+  res.last_state = std::move(state);
+
   return std::move(res);
 }
 
@@ -1313,12 +1334,11 @@ void TonlibClient::set_config(FullConfig full_config) {
   config_ = std::move(full_config.config);
   config_generation_++;
   wallet_id_ = full_config.wallet_id;
-  blockchain_name_ = config_.zero_state_id.root_hash.as_slice().str();
+  last_state_key_ = full_config.last_state_key;
 
   use_callbacks_for_network_ = full_config.use_callbacks_for_network;
-  ignore_cache_ = full_config.ignore_cache;
   init_ext_client();
-  init_last_block(std::move(full_config.o_master_config));
+  init_last_block(std::move(full_config.last_state));
   init_last_config();
   client_.set_client(get_client_ref());
 }
@@ -1332,23 +1352,23 @@ td::Status TonlibClient::do_request(const tonlib_api::close& request,
   return td::Status::OK();
 }
 
-tonlib_api::object_ptr<tonlib_api::Object> TonlibClient::do_static_request(
-    tonlib_api::options_validateConfig& request) {
-  auto r_config = validate_config(std::move(request.config_));
-  if (r_config.is_error()) {
-    return status_to_tonlib_api(r_config.move_as_error());
-  }
-  return tonlib_api::make_object<tonlib_api::options_configInfo>(r_config.ok().wallet_id);
+td::Status TonlibClient::do_request(tonlib_api::options_validateConfig& request,
+                                    td::Promise<object_ptr<tonlib_api::options_configInfo>>&& promise) {
+  TRY_RESULT(config, validate_config(std::move(request.config_)));
+  auto res = to_tonlib_api(config);
+  promise.set_value(std::move(res));
+  return td::Status::OK();
 }
 
 td::Status TonlibClient::do_request(tonlib_api::options_setConfig& request,
-                                    td::Promise<object_ptr<tonlib_api::ok>>&& promise) {
+                                    td::Promise<object_ptr<tonlib_api::options_configInfo>>&& promise) {
   if (!request.config_) {
     return TonlibError::EmptyField("config");
   }
   TRY_RESULT(config, validate_config(std::move(request.config_)));
+  auto res = to_tonlib_api(config);
   set_config(std::move(config));
-  promise.set_value(tonlib_api::make_object<tonlib_api::ok>());
+  promise.set_value(std::move(res));
   return td::Status::OK();
 }
 
@@ -1603,11 +1623,9 @@ td::Result<KeyStorage::InputKey> from_tonlib(tonlib_api::inputKeyRegular& input_
 }
 
 td::Result<KeyStorage::InputKey> from_tonlib(tonlib_api::InputKey& input_key) {
-  td::Result<KeyStorage::InputKey> r_key;
-  tonlib_api::downcast_call(
-      input_key, td::overloaded([&](tonlib_api::inputKeyRegular& input_key) { r_key = from_tonlib(input_key); },
-                                [&](tonlib_api::inputKeyFake&) { r_key = KeyStorage::fake_input_key(); }));
-  return r_key;
+  return downcast_call2<td::Result<KeyStorage::InputKey>>(
+      input_key, td::overloaded([&](tonlib_api::inputKeyRegular& input_key) { return from_tonlib(input_key); },
+                                [&](tonlib_api::inputKeyFake&) { return KeyStorage::fake_input_key(); }));
 }
 
 // ton::TestWallet
@@ -2195,6 +2213,98 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getState& request,
   return td::Status::OK();
 }
 
+bool is_list(vm::StackEntry entry) {
+  while (true) {
+    if (entry.type() == vm::StackEntry::Type::t_null) {
+      return true;
+    }
+    if (entry.type() != vm::StackEntry::Type::t_tuple) {
+      return false;
+    }
+    if (entry.as_tuple()->size() != 2) {
+      return false;
+    }
+    entry = entry.as_tuple()->at(1);
+  }
+};
+auto to_tonlib_api(const vm::StackEntry& entry) -> tonlib_api::object_ptr<tonlib_api::tvm_StackEntry> {
+  switch (entry.type()) {
+    case vm::StackEntry::Type::t_int:
+      return tonlib_api::make_object<tonlib_api::tvm_stackEntryNumber>(
+          tonlib_api::make_object<tonlib_api::tvm_numberDecimal>(dec_string(entry.as_int())));
+    case vm::StackEntry::Type::t_slice:
+      return tonlib_api::make_object<tonlib_api::tvm_stackEntryCell>(tonlib_api::make_object<tonlib_api::tvm_cell>(
+          to_bytes(vm::CellBuilder().append_cellslice(entry.as_slice()).finalize())));
+    case vm::StackEntry::Type::t_cell:
+      return tonlib_api::make_object<tonlib_api::tvm_stackEntryCell>(
+          tonlib_api::make_object<tonlib_api::tvm_cell>(to_bytes(entry.as_cell())));
+    case vm::StackEntry::Type::t_null:
+    case vm::StackEntry::Type::t_tuple: {
+      std::vector<tonlib_api::object_ptr<tonlib_api::tvm_StackEntry>> elements;
+      if (is_list(entry)) {
+        auto node = entry;
+        while (node.type() == vm::StackEntry::Type::t_tuple) {
+          elements.push_back(to_tonlib_api(node.as_tuple()->at(0)));
+          node = node.as_tuple()->at(1);
+        }
+        return tonlib_api::make_object<tonlib_api::tvm_stackEntryList>(
+            tonlib_api::make_object<tonlib_api::tvm_list>(std::move(elements)));
+
+      } else {
+        for (auto& element : *entry.as_tuple()) {
+          elements.push_back(to_tonlib_api(element));
+        }
+        return tonlib_api::make_object<tonlib_api::tvm_stackEntryTuple>(
+            tonlib_api::make_object<tonlib_api::tvm_tuple>(std::move(elements)));
+      }
+    }
+
+    default:
+      return tonlib_api::make_object<tonlib_api::tvm_stackEntryUnsupported>();
+  }
+};
+
+td::Result<vm::StackEntry> from_tonlib_api(tonlib_api::tvm_StackEntry& entry) {
+  // TODO: error codes
+  // downcast_call
+  return downcast_call2<td::Result<vm::StackEntry>>(
+      entry,
+      td::overloaded(
+          [&](tonlib_api::tvm_stackEntryUnsupported& cell) { return td::Status::Error("Unsuppored stack entry"); },
+          [&](tonlib_api::tvm_stackEntrySlice& cell) -> td::Result<vm::StackEntry> {
+            TRY_RESULT(res, vm::std_boc_deserialize(cell.slice_->bytes_));
+            return vm::StackEntry{std::move(res)};
+          },
+          [&](tonlib_api::tvm_stackEntryCell& cell) -> td::Result<vm::StackEntry> {
+            TRY_RESULT(res, vm::std_boc_deserialize(cell.cell_->bytes_));
+            return vm::StackEntry{std::move(res)};
+          },
+          [&](tonlib_api::tvm_stackEntryTuple& tuple) -> td::Result<vm::StackEntry> {
+            std::vector<vm::StackEntry> elements;
+            for (auto& element : tuple.tuple_->elements_) {
+              TRY_RESULT(new_element, from_tonlib_api(*element));
+              elements.push_back(std::move(new_element));
+            }
+            return td::Ref<vm::Tuple>(true, std::move(elements));
+          },
+          [&](tonlib_api::tvm_stackEntryList& tuple) -> td::Result<vm::StackEntry> {
+            vm::StackEntry tail;
+            for (auto& element : td::reversed(tuple.list_->elements_)) {
+              TRY_RESULT(new_element, from_tonlib_api(*element));
+              tail = vm::make_tuple_ref(std::move(new_element), std::move(tail));
+            }
+            return tail;
+          },
+          [&](tonlib_api::tvm_stackEntryNumber& number) -> td::Result<vm::StackEntry> {
+            auto& dec = *number.number_;
+            auto num = td::dec_string_to_int256(dec.number_);
+            if (num.is_null()) {
+              return td::Status::Error("Failed to parse dec string to int256");
+            }
+            return num;
+          }));
+}
+
 td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request,
                                     td::Promise<object_ptr<tonlib_api::smc_runResult>>&& promise) {
   auto it = smcs_.find(request.id_);
@@ -2209,65 +2319,17 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request,
                                [&](tonlib_api::smc_methodIdName& name) { args.set_method_id(name.name_); }));
   td::Ref<vm::Stack> stack(true);
   td::Status status;
-  // TODO: error codes
-  // downcast_call
   for (auto& entry : request.stack_) {
-    downcast_call(*entry, td::overloaded(
-                              [&](tonlib_api::tvm_stackEntryUnsupported& cell) {
-                                status = td::Status::Error("Unsuppored stack entry");
-                              },
-                              [&](tonlib_api::tvm_stackEntrySlice& cell) {
-                                auto r_cell = vm::std_boc_deserialize(cell.slice_->bytes_);
-                                if (r_cell.is_error()) {
-                                  status = r_cell.move_as_error();
-                                  return;
-                                }
-                                stack.write().push_cell(r_cell.move_as_ok());
-                              },
-                              [&](tonlib_api::tvm_stackEntryCell& cell) {
-                                auto r_cell = vm::std_boc_deserialize(cell.cell_->bytes_);
-                                if (r_cell.is_error()) {
-                                  status = r_cell.move_as_error();
-                                  return;
-                                }
-                                stack.write().push_cell(r_cell.move_as_ok());
-                              },
-                              [&](tonlib_api::tvm_stackEntryNumber& number) {
-                                [&](tonlib_api::tvm_numberDecimal& dec) {
-                                  auto num = td::dec_string_to_int256(dec.number_);
-                                  if (num.is_null()) {
-                                    status = td::Status::Error("Failed to parse dec string to int256");
-                                    return;
-                                  }
-                                  stack.write().push_int(std::move(num));
-                                }(*number.number_);
-                              }));
+    TRY_RESULT(e, from_tonlib_api(*entry));
+    stack.write().push(std::move(e));
   }
-  TRY_STATUS(std::move(status));
   args.set_stack(std::move(stack));
   auto res = smc->run_get_method(std::move(args));
 
   // smc.runResult gas_used:int53 stack:vector<tvm.StackEntry> exit_code:int32 = smc.RunResult;
   std::vector<object_ptr<tonlib_api::tvm_StackEntry>> res_stack;
   for (auto& entry : res.stack->as_span()) {
-    switch (entry.type()) {
-      case vm::StackEntry::Type::t_int:
-        res_stack.push_back(tonlib_api::make_object<tonlib_api::tvm_stackEntryNumber>(
-            tonlib_api::make_object<tonlib_api::tvm_numberDecimal>(dec_string(entry.as_int()))));
-        break;
-      case vm::StackEntry::Type::t_slice:
-        res_stack.push_back(
-            tonlib_api::make_object<tonlib_api::tvm_stackEntryCell>(tonlib_api::make_object<tonlib_api::tvm_cell>(
-                to_bytes(vm::CellBuilder().append_cellslice(entry.as_slice()).finalize()))));
-        break;
-      case vm::StackEntry::Type::t_cell:
-        res_stack.push_back(tonlib_api::make_object<tonlib_api::tvm_stackEntryCell>(
-            tonlib_api::make_object<tonlib_api::tvm_cell>(to_bytes(entry.as_cell()))));
-        break;
-      default:
-        res_stack.push_back(tonlib_api::make_object<tonlib_api::tvm_stackEntryUnsupported>());
-        break;
-    }
+    res_stack.push_back(to_tonlib_api(entry));
   }
   promise.set_value(tonlib_api::make_object<tonlib_api::smc_runResult>(res.gas_used, std::move(res_stack), res.code));
   return td::Status::OK();
@@ -2576,11 +2638,6 @@ td::Status TonlibClient::do_request(const tonlib_api::unpackAccountAddress& requ
   return TonlibError::Internal();
 }
 template <class P>
-td::Status TonlibClient::do_request(const tonlib_api::options_validateConfig& request, P&&) {
-  UNREACHABLE();
-  return TonlibError::Internal();
-}
-template <class P>
 td::Status TonlibClient::do_request(tonlib_api::getBip39Hints& request, P&&) {
   UNREACHABLE();
   return TonlibError::Internal();

tonlib/tonlib/TonlibClient.h

@@ -58,6 +58,14 @@ class TonlibClient : public td::actor::Actor {
 
   ~TonlibClient();
 
+  struct FullConfig {
+    Config config;
+    bool use_callbacks_for_network;
+    LastBlockState last_state;
+    std::string last_state_key;
+    td::uint32 wallet_id;
+  };
+
  private:
   enum class State { Uninited, Running, Closed } state_ = State::Uninited;
   td::unique_ptr<TonlibCallback> callback_;
@@ -66,8 +74,7 @@ class TonlibClient : public td::actor::Actor {
   Config config_;
   td::uint32 config_generation_{0};
   td::uint32 wallet_id_;
-  std::string blockchain_name_;
-  bool ignore_cache_{false};
+  std::string last_state_key_;
   bool use_callbacks_for_network_{false};
 
   // KeyStorage
@@ -89,7 +96,7 @@ class TonlibClient : public td::actor::Actor {
 
   ExtClientRef get_client_ref();
   void init_ext_client();
-  void init_last_block(td::optional<Config> o_master_config);
+  void init_last_block(LastBlockState state);
   void init_last_config();
 
   bool is_closing_{false};
@@ -128,7 +135,6 @@ class TonlibClient : public td::actor::Actor {
   static object_ptr<tonlib_api::Object> do_static_request(const tonlib_api::testGiver_getAccountAddress& request);
   static object_ptr<tonlib_api::Object> do_static_request(const tonlib_api::packAccountAddress& request);
   static object_ptr<tonlib_api::Object> do_static_request(const tonlib_api::unpackAccountAddress& request);
-  static object_ptr<tonlib_api::Object> do_static_request(tonlib_api::options_validateConfig& request);
   static object_ptr<tonlib_api::Object> do_static_request(tonlib_api::getBip39Hints& request);
 
   static object_ptr<tonlib_api::Object> do_static_request(tonlib_api::setLogStream& request);
@@ -161,8 +167,6 @@ class TonlibClient : public td::actor::Actor {
   template <class P>
   td::Status do_request(const tonlib_api::unpackAccountAddress& request, P&&);
   template <class P>
-  td::Status do_request(const tonlib_api::options_validateConfig& request, P&&);
-  template <class P>
   td::Status do_request(tonlib_api::getBip39Hints& request, P&&);
 
   template <class P>
@@ -197,18 +201,14 @@ class TonlibClient : public td::actor::Actor {
     }
   }
 
-  struct FullConfig {
-    Config config;
-    td::optional<Config> o_master_config;
-    bool use_callbacks_for_network;
-    bool ignore_cache;
-    td::uint32 wallet_id;
-  };
-  static td::Result<FullConfig> validate_config(tonlib_api::object_ptr<tonlib_api::config> config);
+  td::Result<FullConfig> validate_config(tonlib_api::object_ptr<tonlib_api::config> config);
   void set_config(FullConfig config);
-  td::Status do_request(const tonlib_api::init& request, td::Promise<object_ptr<tonlib_api::ok>>&& promise);
+  td::Status do_request(const tonlib_api::init& request, td::Promise<object_ptr<tonlib_api::options_info>>&& promise);
   td::Status do_request(const tonlib_api::close& request, td::Promise<object_ptr<tonlib_api::ok>>&& promise);
-  td::Status do_request(tonlib_api::options_setConfig& request, td::Promise<object_ptr<tonlib_api::ok>>&& promise);
+  td::Status do_request(tonlib_api::options_validateConfig& request,
+                        td::Promise<object_ptr<tonlib_api::options_configInfo>>&& promise);
+  td::Status do_request(tonlib_api::options_setConfig& request,
+                        td::Promise<object_ptr<tonlib_api::options_configInfo>>&& promise);
 
   td::Status do_request(const tonlib_api::raw_sendMessage& request, td::Promise<object_ptr<tonlib_api::ok>>&& promise);
   td::Status do_request(const tonlib_api::raw_createAndSendMessage& request,

tonlib/tonlib/tonlib-cli.cpp

@@ -2,6 +2,7 @@
 
 #include "td/utils/filesystem.h"
 #include "td/utils/OptionsParser.h"
+#include "td/utils/overloaded.h"
 #include "td/utils/Parser.h"
 #include "td/utils/port/signals.h"
 #include "td/utils/port/path.h"
@@ -177,10 +178,6 @@ class TonlibCli : public td::actor::Actor {
                       ? make_object<tonlib_api::config>(options_.config, options_.name,
                                                         options_.use_callbacks_for_network, options_.ignore_cache)
                       : nullptr;
-    auto config2 = !options_.config.empty()
-                       ? make_object<tonlib_api::config>(options_.config, options_.name,
-                                                         options_.use_callbacks_for_network, options_.ignore_cache)
-                       : nullptr;
 
     tonlib_api::object_ptr<tonlib_api::KeyStoreType> ks_type;
     if (options_.in_memory) {
@@ -188,18 +185,13 @@ class TonlibCli : public td::actor::Actor {
     } else {
       ks_type = make_object<tonlib_api::keyStoreTypeDirectory>(options_.key_dir);
     }
-    auto obj =
-        tonlib::TonlibClient::static_request(make_object<tonlib_api::options_validateConfig>(std::move(config2)));
-    if (obj->get_id() != tonlib_api::error::ID) {
-      auto info = ton::move_tl_object_as<tonlib_api::options_configInfo>(obj);
-      wallet_id_ = static_cast<td::uint32>(info->default_wallet_id_);
-    } else {
-      LOG(ERROR) << "Invalid config";
-    }
     send_query(make_object<tonlib_api::init>(make_object<tonlib_api::options>(std::move(config), std::move(ks_type))),
-               [](auto r_ok) {
+               [&](auto r_ok) {
                  LOG_IF(ERROR, r_ok.is_error()) << r_ok.error();
-                 td::TerminalIO::out() << "Tonlib is inited\n";
+                 if (r_ok.is_ok()) {
+                   wallet_id_ = static_cast<td::uint32>(r_ok.ok()->config_info_->default_wallet_id_);
+                   td::TerminalIO::out() << "Tonlib is inited\n";
+                 }
                });
     if (options_.one_shot) {
       td::actor::send_closure(actor_id(this), &TonlibCli::parse_line, td::BufferSlice(options_.cmd));
@@ -482,6 +474,73 @@ class TonlibCli : public td::actor::Actor {
         tonlib_api::make_object<tonlib_api::tvm_numberDecimal>(dec_string(num)));
   }
 
+  td::Result<std::vector<tonlib_api::object_ptr<tonlib_api::tvm_StackEntry>>> parse_stack(td::ConstParser& parser,
+                                                                                          td::Slice end_token) {
+    std::vector<tonlib_api::object_ptr<tonlib_api::tvm_StackEntry>> stack;
+    while (true) {
+      auto word = parser.read_word();
+      LOG(ERROR) << word << " vs " << end_token;
+      if (word == end_token) {
+        break;
+      }
+      if (word == "[") {
+        TRY_RESULT(elements, parse_stack(parser, "]"));
+        stack.push_back(tonlib_api::make_object<tonlib_api::tvm_stackEntryTuple>(
+            tonlib_api::make_object<tonlib_api::tvm_tuple>(std::move(elements))));
+      } else if (word == "(") {
+        TRY_RESULT(elements, parse_stack(parser, ")"));
+        stack.push_back(tonlib_api::make_object<tonlib_api::tvm_stackEntryList>(
+            tonlib_api::make_object<tonlib_api::tvm_list>(std::move(elements))));
+      } else {
+        TRY_RESULT(stack_entry, parse_stack_entry(word));
+        stack.push_back(std::move(stack_entry));
+      }
+    }
+    return std::move(stack);
+  }
+
+  static void store_entry(td::StringBuilder& sb, tonlib_api::tvm_StackEntry& entry) {
+    downcast_call(entry, td::overloaded(
+                             [&](tonlib_api::tvm_stackEntryCell& cell) {
+                               auto r_cell = vm::std_boc_deserialize(cell.cell_->bytes_);
+                               if (r_cell.is_error()) {
+                                 sb << "<INVALID_CELL>";
+                               }
+                               auto cs = vm::load_cell_slice(r_cell.move_as_ok());
+                               std::stringstream ss;
+                               cs.print_rec(ss);
+                               sb << ss.str();
+                             },
+                             [&](tonlib_api::tvm_stackEntrySlice& cell) {
+                               auto r_cell = vm::std_boc_deserialize(cell.slice_->bytes_);
+                               if (r_cell.is_error()) {
+                                 sb << "<INVALID_CELL>";
+                               }
+                               auto cs = vm::load_cell_slice(r_cell.move_as_ok());
+                               std::stringstream ss;
+                               cs.print_rec(ss);
+                               sb << ss.str();
+                             },
+                             [&](tonlib_api::tvm_stackEntryNumber& cell) { sb << cell.number_->number_; },
+                             [&](tonlib_api::tvm_stackEntryTuple& cell) {
+                               sb << "(";
+                               for (auto& element : cell.tuple_->elements_) {
+                                 sb << " ";
+                                 store_entry(sb, *element);
+                               }
+                               sb << " )";
+                             },
+                             [&](tonlib_api::tvm_stackEntryList& cell) {
+                               sb << "[";
+                               for (auto& element : cell.list_->elements_) {
+                                 sb << " ";
+                                 store_entry(sb, *element);
+                               }
+                               sb << " ]";
+                             },
+                             [&](tonlib_api::tvm_stackEntryUnsupported& cell) { sb << "<UNSUPPORTED>"; }));
+  }
+
   void run_method(td::ConstParser& parser, td::Promise<td::Unit> promise) {
     TRY_RESULT_PROMISE(promise, addr, to_account_address(parser.read_word(), false));
 
@@ -492,15 +551,15 @@ class TonlibCli : public td::actor::Actor {
     } else {
       method = tonlib_api::make_object<tonlib_api::smc_methodIdName>(method_str.str());
     }
-    std::vector<tonlib_api::object_ptr<tonlib_api::tvm_StackEntry>> stack;
-    while (true) {
-      auto word = parser.read_word();
-      if (word.empty()) {
-        break;
-      }
-      TRY_RESULT_PROMISE(promise, stack_entry, parse_stack_entry(word));
-      stack.push_back(std::move(stack_entry));
+    TRY_RESULT_PROMISE(promise, stack, parse_stack(parser, ""));
+    td::StringBuilder sb;
+    for (auto& entry : stack) {
+      store_entry(sb, *entry);
+      sb << "\n";
     }
+
+    td::TerminalIO::out() << "Run " << to_string(method) << "With stack:\n" << sb.as_cslice();
+
     auto to_run =
         tonlib_api::make_object<tonlib_api::smc_runGetMethod>(0 /*fixme*/, std::move(method), std::move(stack));
 
@@ -513,9 +572,16 @@ class TonlibCli : public td::actor::Actor {
     to_run->id_ = info->id_;
     send_query(std::move(to_run), promise.send_closure(actor_id(this), &TonlibCli::run_method_3));
   }
-
   void run_method_3(tonlib_api::object_ptr<tonlib_api::smc_runResult> info, td::Promise<td::Unit> promise) {
-    td::TerminalIO::out() << "Got smc result " << to_string(info);
+    td::StringBuilder sb;
+    for (auto& entry : info->stack_) {
+      store_entry(sb, *entry);
+      sb << "\n";
+    }
+
+    td::TerminalIO::out() << "Got smc result. exit code: " << info->exit_code_ << ", gas_used: " << info->gas_used_
+                          << "\n"
+                          << sb.as_cslice();
     promise.set_value({});
   }
 
@@ -972,7 +1038,7 @@ class TonlibCli : public td::actor::Actor {
 
   void import_key(std::vector<td::SecureString> words, td::Slice password) {
     using tonlib_api::make_object;
-    send_query(make_object<tonlib_api::importKey>(td::SecureString(password), td::SecureString(),
+    send_query(make_object<tonlib_api::importKey>(td::SecureString(password), td::SecureString(" test mnemonic"),
                                                   make_object<tonlib_api::exportedKey>(std::move(words))),
                [this, password = td::SecureString(password)](auto r_res) {
                  if (r_res.is_error()) {
@@ -1147,7 +1213,7 @@ class TonlibCli : public td::actor::Actor {
                                       return;
                                     }
                                     td::TerminalIO::out() << to_string(r_res.ok());
-                                    self->on_ok();
+                                    //self->on_ok();
                                   });
 
                  self->send_query(make_object<tonlib_api::query_send>(r_res.ok()->id_), [self](auto r_res) {
@@ -1160,7 +1226,7 @@ class TonlibCli : public td::actor::Actor {
                    self->on_ok();
                  });
 
-                 self->on_ok();
+                 //self->on_ok();
                });
   }
 

utils/CMakeLists.txt

@@ -14,3 +14,8 @@ add_executable(json2tlo json2tlo.cpp )
 target_link_libraries(json2tlo tl_api ton_crypto keys )
 target_include_directories(json2tlo PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/..)
+
+add_executable(pack-viewer pack-viewer.cpp )
+target_link_libraries(pack-viewer tl_api ton_crypto keys validator tddb )
+target_include_directories(pack-viewer PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/..)

utils/pack-viewer.cpp

@@ -0,0 +1,61 @@
+/* 
+    This file is part of TON Blockchain source code.
+
+    TON Blockchain is free software; you can redistribute it and/or
+    modify it under the terms of the GNU General Public License
+    as published by the Free Software Foundation; either version 2
+    of the License, or (at your option) any later version.
+
+    TON Blockchain is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with TON Blockchain.  If not, see <http://www.gnu.org/licenses/>.
+
+    In addition, as a special exception, the copyright holders give permission 
+    to link the code of portions of this program with the OpenSSL library. 
+    You must obey the GNU General Public License in all respects for all 
+    of the code used other than OpenSSL. If you modify file(s) with this 
+    exception, you may extend this exception to your version of the file(s), 
+    but you are not obligated to do so. If you do not wish to do so, delete this 
+    exception statement from your version. If you delete this exception statement 
+    from all source files in the program, then also delete it here.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include <iostream>
+#include <iomanip>
+#include <string>
+#include <cstring>
+#include <cassert>
+#include "td/utils/OptionsParser.h"
+
+#include "validator/db/package.hpp"
+#include "validator/db/fileref.hpp"
+
+void run(std::string filename) {
+  auto R = ton::Package::open(filename, true, false);
+  if (R.is_error()) {
+    std::cerr << "failed to open archive '" << filename << "': " << R.move_as_error().to_string();
+    std::_Exit(2);
+  }
+  auto p = R.move_as_ok();
+
+  p.iterate([&](std::string filename, td::BufferSlice data, td::uint64 offset) -> bool {
+    auto E = ton::validator::FileReference::create(filename);
+    if (E.is_error()) {
+      std::cout << "bad filename\n";
+    } else {
+      std::cout << filename << " " << data.size() << "\n";
+    }
+    return true;
+  });
+}
+
+int main(int argc, char **argv) {
+  run(argv[1]);
+  return 0;
+}
+

validator-engine/validator-engine.cpp

@@ -43,6 +43,7 @@
 #include "td/utils/port/path.h"
 #include "td/utils/port/signals.h"
 #include "td/utils/port/user.h"
+#include "td/utils/port/rlimit.h"
 #include "td/utils/ThreadSafeCounter.h"
 #include "td/utils/TsFileLog.h"
 #include "td/utils/Random.h"
@@ -2842,6 +2843,8 @@ int main(int argc, char *argv[]) {
     td::log_interface = td::default_log_interface;
   };
 
+  LOG_STATUS(td::change_maximize_rlimit(td::RlimitType::nofile, 65536));
+
   std::vector<std::function<void()>> acts;
 
   td::OptionsParser p;

validator/CMakeLists.txt

@@ -10,17 +10,17 @@ add_subdirectory(impl)
 set(VALIDATOR_DB_SOURCE
   db/archiver.cpp
   db/archiver.hpp
-  db/archive-db.cpp
-  db/archive-db.hpp
+  db/archive-manager.cpp
+  db/archive-manager.hpp
+  db/archive-slice.cpp
+  db/archive-slice.hpp
   db/blockdb.cpp
   db/blockdb.hpp
   db/celldb.cpp
   db/celldb.hpp
   db/files-async.hpp
-  db/filedb.hpp
-  db/filedb.cpp
-  db/ltdb.hpp
-  db/ltdb.cpp
+  db/fileref.hpp
+  db/fileref.cpp
   db/rootdb.cpp
   db/rootdb.hpp
   db/statedb.hpp
@@ -52,6 +52,8 @@ set(VALIDATOR_HEADERS
   interfaces/validator-manager.h
   interfaces/validator-set.h
   invariants.hpp
+  
+  import-db-slice.hpp
 
   manager-disk.h
   manager-disk.hpp
@@ -69,6 +71,7 @@ set(VALIDATOR_SOURCE
   apply-block.cpp
   block-handle.cpp
   get-next-key-blocks.cpp
+  import-db-slice.cpp
   shard-client.cpp
   state-serializer.cpp
   token-manager.cpp
@@ -124,6 +127,8 @@ set(FULL_NODE_SOURCE
   net/download-block.cpp
   net/download-block-new.hpp
   net/download-block-new.cpp
+  net/download-archive-slice.hpp
+  net/download-archive-slice.cpp
   net/download-next-block.hpp
   net/download-next-block.cpp
   net/download-state.hpp

validator/apply-block.cpp

@@ -53,6 +53,10 @@ void ApplyBlock::alarm() {
 void ApplyBlock::start_up() {
   VLOG(VALIDATOR_DEBUG) << "running apply_block for " << id_;
 
+  if (id_.is_masterchain()) {
+    masterchain_block_id_ = id_;
+  }
+
   alarm_timestamp() = timeout_;
 
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
@@ -144,6 +148,7 @@ void ApplyBlock::got_block_handle(BlockHandle handle) {
 }
 
 void ApplyBlock::written_block_data() {
+  VLOG(VALIDATOR_DEBUG) << "apply block: written block data for " << id_;
   if (handle_->is_applied() && handle_->processed()) {
     finish_query();
   } else {
@@ -161,6 +166,7 @@ void ApplyBlock::written_block_data() {
 }
 
 void ApplyBlock::got_cur_state(td::Ref<ShardState> state) {
+  VLOG(VALIDATOR_DEBUG) << "apply block: received state for " << id_;
   state_ = std::move(state);
   CHECK(handle_->received_state());
   written_state();
@@ -171,6 +177,7 @@ void ApplyBlock::written_state() {
     finish_query();
     return;
   }
+  VLOG(VALIDATOR_DEBUG) << "apply block: setting next for parents of " << id_;
 
   if (handle_->id().id.seqno != 0 && !handle_->is_applied()) {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
@@ -201,10 +208,12 @@ void ApplyBlock::written_next() {
     return;
   }
 
-  if (handle_->id().id.seqno != 0) {
+  VLOG(VALIDATOR_DEBUG) << "apply block: applying parents of " << id_;
+
+  if (handle_->id().id.seqno != 0 && !handle_->is_applied()) {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
       if (R.is_error()) {
-        td::actor::send_closure(SelfId, &ApplyBlock::abort_query, R.move_as_error());
+        td::actor::send_closure(SelfId, &ApplyBlock::abort_query, R.move_as_error_prefix("prev: "));
       } else {
         td::actor::send_closure(SelfId, &ApplyBlock::applied_prev);
       }
@@ -213,9 +222,13 @@ void ApplyBlock::written_next() {
     td::MultiPromise mp;
     auto g = mp.init_guard();
     g.add_promise(std::move(P));
-    run_apply_block_query(handle_->one_prev(true), td::Ref<BlockData>{}, manager_, timeout_, g.get_promise());
+    BlockIdExt m = masterchain_block_id_;
+    if (id_.is_masterchain()) {
+      m = id_;
+    }
+    run_apply_block_query(handle_->one_prev(true), td::Ref<BlockData>{}, m, manager_, timeout_, g.get_promise());
     if (handle_->merge_before()) {
-      run_apply_block_query(handle_->one_prev(false), td::Ref<BlockData>{}, manager_, timeout_, g.get_promise());
+      run_apply_block_query(handle_->one_prev(false), td::Ref<BlockData>{}, m, manager_, timeout_, g.get_promise());
     }
   } else {
     applied_prev();
@@ -223,6 +236,10 @@ void ApplyBlock::written_next() {
 }
 
 void ApplyBlock::applied_prev() {
+  VLOG(VALIDATOR_DEBUG) << "apply block: waiting manager's confirm for " << id_;
+  if (!id_.is_masterchain()) {
+    handle_->set_masterchain_ref_block(masterchain_block_id_.seqno());
+  }
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
     if (R.is_error()) {
       td::actor::send_closure(SelfId, &ApplyBlock::abort_query, R.move_as_error());
@@ -234,7 +251,12 @@ void ApplyBlock::applied_prev() {
 }
 
 void ApplyBlock::applied_set() {
+  VLOG(VALIDATOR_DEBUG) << "apply block: setting apply bit for " << id_;
   handle_->set_applied();
+  if (handle_->id().seqno() > 0) {
+    CHECK(handle_->handle_moved_to_archive());
+    CHECK(handle_->moved_to_archive());
+  }
   if (handle_->need_flush()) {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
       if (R.is_error()) {

validator/apply-block.hpp

@@ -39,9 +39,14 @@ namespace validator {
 
 class ApplyBlock : public td::actor::Actor {
  public:
-  ApplyBlock(BlockIdExt id, td::Ref<BlockData> block, td::actor::ActorId<ValidatorManager> manager,
-             td::Timestamp timeout, td::Promise<td::Unit> promise)
-      : id_(id), block_(std::move(block)), manager_(manager), timeout_(timeout), promise_(std::move(promise)) {
+  ApplyBlock(BlockIdExt id, td::Ref<BlockData> block, BlockIdExt masterchain_block_id,
+             td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout, td::Promise<td::Unit> promise)
+      : id_(id)
+      , block_(std::move(block))
+      , masterchain_block_id_(masterchain_block_id)
+      , manager_(manager)
+      , timeout_(timeout)
+      , promise_(std::move(promise)) {
   }
 
   static constexpr td::uint32 apply_block_priority() {
@@ -65,6 +70,7 @@ class ApplyBlock : public td::actor::Actor {
  private:
   BlockIdExt id_;
   td::Ref<BlockData> block_;
+  BlockIdExt masterchain_block_id_;
   td::actor::ActorId<ValidatorManager> manager_;
   td::Timestamp timeout_;
   td::Promise<td::Unit> promise_;

validator/block-handle.cpp

@@ -32,18 +32,19 @@ void BlockHandleImpl::flush(td::actor::ActorId<ValidatorManagerInterface> manage
 td::BufferSlice BlockHandleImpl::serialize() const {
   while (locked()) {
   }
-  auto flags = flags_.load(std::memory_order_consume) & ~Flags::dbf_processed;
+  auto flags = flags_.load(std::memory_order_consume) & ~(Flags::dbf_processed | Flags::dbf_moved_handle);
   return create_serialize_tl_object<ton_api::db_block_info>(
       create_tl_block_id(id_), flags, (flags & dbf_inited_prev_left) ? create_tl_block_id(prev_[0]) : nullptr,
       (flags & dbf_inited_prev_right) ? create_tl_block_id(prev_[1]) : nullptr,
       (flags & dbf_inited_next_left) ? create_tl_block_id(next_[0]) : nullptr,
       (flags & dbf_inited_next_right) ? create_tl_block_id(next_[1]) : nullptr, (flags & dbf_inited_lt) ? lt_ : 0,
-      (flags & dbf_inited_ts) ? ts_ : 0, (flags & dbf_inited_state) ? state_ : RootHash::zero());
+      (flags & dbf_inited_ts) ? ts_ : 0, (flags & dbf_inited_state) ? state_ : RootHash::zero(),
+      (flags & dbf_inited_masterchain_ref_block) ? masterchain_ref_seqno_ : 0);
 }
 
 BlockHandleImpl::BlockHandleImpl(td::BufferSlice data) {
   auto obj = fetch_tl_object<ton_api::db_block_info>(std::move(data), true).move_as_ok();
-  flags_ = obj->flags_ & ~Flags::dbf_processed;
+  flags_ = obj->flags_ & ~(Flags::dbf_processed | Flags::dbf_moved_handle);
   id_ = create_block_id(obj->id_);
   prev_[0] = (flags_ & dbf_inited_prev_left) ? create_block_id(obj->prev_left_) : BlockIdExt{};
   prev_[1] = (flags_ & dbf_inited_prev_right) ? create_block_id(obj->prev_right_) : BlockIdExt{};
@@ -52,6 +53,8 @@ BlockHandleImpl::BlockHandleImpl(td::BufferSlice data) {
   lt_ = (flags_ & dbf_inited_lt) ? obj->lt_ : 0;
   ts_ = (flags_ & dbf_inited_ts) ? obj->ts_ : 0;
   state_ = (flags_ & dbf_inited_state) ? obj->state_ : RootHash::zero();
+  masterchain_ref_seqno_ =
+      (flags_ & dbf_inited_masterchain_ref_block) ? static_cast<BlockSeqno>(obj->masterchain_ref_seqno_) : 0;
   get_thread_safe_counter().add(1);
 }
 

validator/block-handle.hpp

@@ -56,11 +56,12 @@ struct BlockHandleImpl : public BlockHandleInterface {
     dbf_inited_state_boc = 0x100000,
     dbf_archived = 0x200000,
     dbf_applied = 0x400000,
-    dbf_moved = 0x1000000,
+    dbf_inited_masterchain_ref_block = 0x800000,
     dbf_deleted = 0x2000000,
     dbf_deleted_boc = 0x4000000,
     dbf_moved_new = 0x8000000,
     dbf_processed = 0x10000000,
+    dbf_moved_handle = 0x20000000,
   };
 
   std::atomic<td::uint64> version_{0};
@@ -72,6 +73,7 @@ struct BlockHandleImpl : public BlockHandleInterface {
   LogicalTime lt_;
   UnixTime ts_;
   RootHash state_;
+  BlockSeqno masterchain_ref_seqno_;
 
   static constexpr td::uint64 lock_const() {
     return static_cast<td::uint64>(1) << 32;
@@ -93,12 +95,12 @@ struct BlockHandleImpl : public BlockHandleInterface {
   bool received() const override {
     return flags_.load(std::memory_order_consume) & Flags::dbf_received;
   }
-  bool moved_to_storage() const override {
-    return flags_.load(std::memory_order_consume) & Flags::dbf_moved;
-  }
   bool moved_to_archive() const override {
     return flags_.load(std::memory_order_consume) & Flags::dbf_moved_new;
   }
+  bool handle_moved_to_archive() const override {
+    return flags_.load(std::memory_order_consume) & Flags::dbf_moved_handle;
+  }
   bool deleted() const override {
     return flags_.load(std::memory_order_consume) & Flags::dbf_deleted;
   }
@@ -199,6 +201,13 @@ struct BlockHandleImpl : public BlockHandleInterface {
   bool is_applied() const override {
     return flags_.load(std::memory_order_consume) & Flags::dbf_applied;
   }
+  bool inited_masterchain_ref_block() const override {
+    return id_.is_masterchain() || (flags_.load(std::memory_order_consume) & Flags::dbf_inited_masterchain_ref_block);
+  }
+  BlockSeqno masterchain_ref_block() const override {
+    CHECK(inited_masterchain_ref_block());
+    return id_.is_masterchain() ? id_.seqno() : masterchain_ref_seqno_;
+  }
   std::vector<BlockIdExt> prev() const override {
     if (is_zero()) {
       return {};
@@ -389,23 +398,17 @@ struct BlockHandleImpl : public BlockHandleInterface {
     flags_ |= Flags::dbf_received;
     unlock();
   }
-  void set_moved_to_storage() override {
-    if (flags_.load(std::memory_order_consume) & Flags::dbf_moved) {
-      return;
-    }
-    lock();
-    flags_ |= Flags::dbf_moved;
-    unlock();
-  }
   void set_moved_to_archive() override {
     if (flags_.load(std::memory_order_consume) & Flags::dbf_moved_new) {
       return;
     }
     lock();
     flags_ |= Flags::dbf_moved_new;
-    flags_ &= ~Flags::dbf_moved;
     unlock();
   }
+  void set_handle_moved_to_archive() override {
+    flags_ |= Flags::dbf_moved_handle;
+  }
   void set_deleted() override {
     if (flags_.load(std::memory_order_consume) & Flags::dbf_deleted) {
       return;
@@ -485,6 +488,14 @@ struct BlockHandleImpl : public BlockHandleInterface {
       unlock();
     }
   }
+  void set_masterchain_ref_block(BlockSeqno seqno) override {
+    if (!inited_masterchain_ref_block()) {
+      lock();
+      masterchain_ref_seqno_ = seqno;
+      flags_ |= Flags::dbf_inited_masterchain_ref_block;
+      unlock();
+    }
+  }
 
   void unsafe_clear_applied() override {
     if (is_applied()) {

validator/db/archive-manager.cpp

@@ -0,0 +1,986 @@
+#include "archive-manager.hpp"
+#include "td/actor/MultiPromise.h"
+#include "td/utils/overloaded.h"
+#include "files-async.hpp"
+#include "td/db/RocksDb.h"
+#include "common/delay.h"
+
+namespace ton {
+
+namespace validator {
+
+std::string PackageId::path() const {
+  if (temp) {
+    return "files/packages/";
+  } else if (key) {
+    char s[24];
+    sprintf(s, "key%03d", id / 1000000);
+    return PSTRING() << "archive/packages/" << s << "/";
+  } else {
+    char s[20];
+    sprintf(s, "arch%04d", id / 100000);
+    return PSTRING() << "archive/packages/" << s << "/";
+  }
+}
+
+std::string PackageId::name() const {
+  if (temp) {
+    return PSTRING() << "temp.archive." << id;
+  } else if (key) {
+    char s[20];
+    sprintf(s, "%06d", id);
+    return PSTRING() << "key.archive." << s;
+  } else {
+    char s[10];
+    sprintf(s, "%05d", id);
+    return PSTRING() << "archive." << s;
+  }
+}
+
+ArchiveManager::ArchiveManager(td::actor::ActorId<RootDb> root, std::string db_root) : db_root_(db_root) {
+}
+
+void ArchiveManager::add_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
+  if (handle->handle_moved_to_archive()) {
+    update_handle(std::move(handle), std::move(promise));
+    return;
+  }
+  auto p = get_package_id_force(handle->masterchain_ref_block(), handle->id().shard_full(), handle->id().seqno(),
+                                handle->unix_time(), handle->logical_time(),
+                                handle->inited_is_key_block() && handle->is_key_block());
+  auto f = get_file_desc(handle->id().shard_full(), p, handle->id().seqno(), handle->unix_time(),
+                         handle->logical_time(), true);
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_handle, std::move(handle), std::move(promise));
+}
+
+void ArchiveManager::update_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
+  FileDescription *f;
+  if (handle->handle_moved_to_archive()) {
+    CHECK(handle->inited_unix_time());
+    f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), handle->id().seqno(),
+                      handle->unix_time(), handle->logical_time(), true);
+  } else {
+    f = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true);
+  }
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::update_handle, std::move(handle), std::move(promise));
+}
+
+void ArchiveManager::add_file(BlockHandle handle, FileReference ref_id, td::BufferSlice data,
+                              td::Promise<td::Unit> promise) {
+  bool copy_to_key = false;
+  if (handle->inited_is_key_block() && handle->is_key_block() && handle->inited_unix_time() &&
+      handle->inited_logical_time() && handle->inited_masterchain_ref_block()) {
+    copy_to_key = (ref_id.ref().get_offset() == ref_id.ref().offset<fileref::Proof>()) ||
+                  (ref_id.ref().get_offset() == ref_id.ref().offset<fileref::ProofLink>());
+  }
+  if (!handle->handle_moved_to_archive()) {
+    td::MultiPromise mp;
+    auto ig = mp.init_guard();
+    ig.add_promise(std::move(promise));
+    auto f1 = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true);
+    td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), data.clone(),
+                            ig.get_promise());
+    if (copy_to_key) {
+      auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()),
+                              handle->id().seqno(), handle->unix_time(), handle->logical_time(), true);
+      td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, ref_id, std::move(data), ig.get_promise());
+    }
+    return;
+  }
+
+  CHECK(handle->inited_is_key_block());
+
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise(std::move(promise));
+  auto f1 = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()),
+                          handle->id().seqno(), handle->unix_time(), handle->logical_time(), true);
+  td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, ref_id, data.clone(), ig.get_promise());
+  if (copy_to_key) {
+    auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()),
+                            handle->id().seqno(), handle->unix_time(), handle->logical_time(), true);
+    td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, ref_id, std::move(data), ig.get_promise());
+  }
+}
+
+void ArchiveManager::add_key_block_proof(UnixTime ts, BlockSeqno seqno, LogicalTime lt, FileReference ref_id,
+                                         td::BufferSlice data, td::Promise<td::Unit> promise) {
+  auto f = get_file_desc(ShardIdFull{masterchainId}, get_key_package_id(seqno), seqno, ts, lt, true);
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), std::move(data),
+                          std::move(promise));
+}
+
+void ArchiveManager::add_temp_file_short(FileReference ref_id, td::BufferSlice data, td::Promise<td::Unit> promise) {
+  auto f = get_file_desc(ref_id.shard(), get_temp_package_id(), 0, 0, 0, true);
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), std::move(data),
+                          std::move(promise));
+}
+
+void ArchiveManager::get_handle(BlockIdExt block_id, td::Promise<BlockHandle> promise) {
+  auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), false);
+  if (f) {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, idx = get_max_temp_file_desc_idx(),
+                                         promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+      if (R.is_ok()) {
+        promise.set_value(R.move_as_ok());
+      } else {
+        td::actor::send_closure(SelfId, &ArchiveManager::get_handle_cont, block_id, idx, std::move(promise));
+      }
+    });
+    td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_handle, block_id, std::move(P));
+  } else {
+    get_handle_cont(block_id, get_max_temp_file_desc_idx(), std::move(promise));
+  }
+}
+
+void ArchiveManager::get_handle_cont(BlockIdExt block_id, PackageId idx, td::Promise<BlockHandle> promise) {
+  if (idx.is_empty()) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "block handle not in db"));
+    return;
+  }
+  auto f = get_temp_file_desc_by_idx(idx);
+  if (!f) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "block handle not in db"));
+    return;
+  }
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, idx = get_prev_temp_file_desc_idx(idx),
+                                       promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+    if (R.is_ok()) {
+      td::actor::send_closure(SelfId, &ArchiveManager::get_handle_finish, R.move_as_ok(), std::move(promise));
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveManager::get_handle_cont, block_id, idx, std::move(promise));
+    }
+  });
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_handle, block_id, std::move(P));
+}
+
+void ArchiveManager::get_handle_finish(BlockHandle handle, td::Promise<BlockHandle> promise) {
+  auto f = get_file_desc_by_seqno(handle->id().shard_full(), handle->id().seqno(), false);
+  if (!f) {
+    promise.set_value(std::move(handle));
+    return;
+  }
+  auto P = td::PromiseCreator::lambda([handle, promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+    if (R.is_ok()) {
+      promise.set_value(R.move_as_ok());
+    } else {
+      promise.set_value(std::move(handle));
+    }
+  });
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_handle, handle->id(), std::move(P));
+}
+
+void ArchiveManager::get_file_short(FileReference ref_id, td::Promise<td::BufferSlice> promise) {
+  bool search_in_key = false;
+  BlockIdExt block_id;
+  ref_id.ref().visit(td::overloaded(
+      [&](const fileref::Proof &p) {
+        search_in_key = p.block_id.is_masterchain();
+        block_id = p.block_id;
+      },
+      [&](const fileref::ProofLink &p) {
+        search_in_key = p.block_id.is_masterchain();
+        block_id = p.block_id;
+      },
+      [&](const auto &p) {}));
+  if (search_in_key) {
+    auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), true);
+    if (f) {
+      auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id,
+                                           promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+        if (R.is_ok()) {
+          promise.set_value(R.move_as_ok());
+        } else {
+          td::actor::send_closure(SelfId, &ArchiveManager::get_temp_file_short, std::move(ref_id), std::move(promise));
+        }
+      });
+      td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, ref_id, std::move(P));
+      return;
+    }
+  }
+  get_temp_file_short(std::move(ref_id), std::move(promise));
+}
+
+void ArchiveManager::get_key_block_proof(FileReference ref_id, td::Promise<td::BufferSlice> promise) {
+  bool search_in_key = false;
+  BlockIdExt block_id;
+  ref_id.ref().visit(td::overloaded(
+      [&](const fileref::Proof &p) {
+        search_in_key = p.block_id.is_masterchain();
+        block_id = p.block_id;
+      },
+      [&](const fileref::ProofLink &p) {
+        search_in_key = p.block_id.is_masterchain();
+        block_id = p.block_id;
+      },
+      [&](const auto &p) {}));
+  if (search_in_key) {
+    auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), true);
+    if (f) {
+      td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, ref_id, std::move(promise));
+    } else {
+      promise.set_error(td::Status::Error(ErrorCode::notready, "key proof not in db"));
+    }
+  } else {
+    promise.set_error(
+        td::Status::Error(ErrorCode::protoviolation, "only proof/prooflink supported in get_key_block_proof"));
+  }
+}
+
+void ArchiveManager::get_temp_file_short(FileReference ref_id, td::Promise<td::BufferSlice> promise) {
+  get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise));
+}
+
+void ArchiveManager::get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise<td::BufferSlice> promise) {
+  auto f = get_temp_file_desc_by_idx(idx);
+  if (!f) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "file not in db"));
+    return;
+  }
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id, idx = get_prev_temp_file_desc_idx(idx),
+                                       promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_ok()) {
+      promise.set_value(R.move_as_ok());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveManager::get_file_short_cont, std::move(ref_id), idx, std::move(promise));
+    }
+  });
+  td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, std::move(ref_id), std::move(P));
+}
+
+void ArchiveManager::get_file(BlockHandle handle, FileReference ref_id, td::Promise<td::BufferSlice> promise) {
+  if (handle->moved_to_archive()) {
+    auto f = get_file_desc(handle->id().shard_full(), PackageId{handle->masterchain_ref_block(), false, false}, 0, 0, 0,
+                           false);
+    if (f) {
+      td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, std::move(ref_id), std::move(promise));
+      return;
+    }
+  }
+  get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise));
+}
+
+void ArchiveManager::written_perm_state(FileReferenceShort id) {
+  perm_states_.emplace(id.hash(), id);
+}
+
+void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise<td::Unit> promise) {
+  auto id = FileReference{fileref::ZeroState{block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) != perm_states_.end()) {
+    promise.set_value(td::Unit());
+    return;
+  }
+
+  auto path = db_root_ + "/archive/states/" + id.filename_short();
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), id = id.shortref(), promise = std::move(promise)](td::Result<std::string> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id);
+          promise.set_value(td::Unit());
+        }
+      });
+  td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/", path, std::move(data), std::move(P))
+      .release();
+}
+
+void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data,
+                                          td::Promise<td::Unit> promise) {
+  auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) != perm_states_.end()) {
+    promise.set_value(td::Unit());
+    return;
+  }
+
+  auto path = db_root_ + "/archive/states/" + id.filename_short();
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), id = id.shortref(), promise = std::move(promise)](td::Result<std::string> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id);
+          promise.set_value(td::Unit());
+        }
+      });
+  td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/", path, std::move(data), std::move(P))
+      .release();
+}
+
+void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {
+  auto id = FileReference{fileref::ZeroState{block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) == perm_states_.end()) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "zerostate not in db"));
+    return;
+  }
+
+  auto path = db_root_ + "/archive/states/" + id.filename_short();
+  td::actor::create_actor<db::ReadFile>("readfile", path, 0, -1, 0, std::move(promise)).release();
+}
+
+void ArchiveManager::check_zero_state(BlockIdExt block_id, td::Promise<bool> promise) {
+  auto id = FileReference{fileref::ZeroState{block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) == perm_states_.end()) {
+    promise.set_result(false);
+    return;
+  }
+  promise.set_result(true);
+}
+
+void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                          td::Promise<td::BufferSlice> promise) {
+  auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) == perm_states_.end()) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db"));
+    return;
+  }
+
+  auto path = db_root_ + "/archive/states/" + id.filename_short();
+  td::actor::create_actor<db::ReadFile>("readfile", path, 0, -1, 0, std::move(promise)).release();
+}
+
+void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,
+                                                td::int64 max_size, td::Promise<td::BufferSlice> promise) {
+  auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) == perm_states_.end()) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db"));
+    return;
+  }
+
+  auto path = db_root_ + "/archive/states/" + id.filename_short();
+  td::actor::create_actor<db::ReadFile>("readfile", path, offset, max_size, 0, std::move(promise)).release();
+}
+
+void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                            td::Promise<bool> promise) {
+  auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
+  auto hash = id.hash();
+  if (perm_states_.find(hash) == perm_states_.end()) {
+    promise.set_result(false);
+    return;
+  }
+  promise.set_result(true);
+}
+
+void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts,
+                                            td::Promise<BlockHandle> promise) {
+  auto f = get_file_desc_by_unix_time(account_id, ts, false);
+  if (f) {
+    td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_unix_time, account_id, ts,
+                            std::move(promise));
+  } else {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "ts not in db"));
+  }
+}
+
+void ArchiveManager::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockHandle> promise) {
+  auto f = get_file_desc_by_lt(account_id, lt, false);
+  if (f) {
+    td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(promise));
+  } else {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "lt not in db"));
+  }
+}
+
+void ArchiveManager::get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno,
+                                        td::Promise<BlockHandle> promise) {
+  auto f = get_file_desc_by_seqno(account_id, seqno, false);
+  if (f) {
+    td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_seqno, account_id, seqno,
+                            std::move(promise));
+  } else {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "seqno not in db"));
+  }
+}
+
+void ArchiveManager::delete_package(PackageId id) {
+  auto key = create_serialize_tl_object<ton_api::db_files_package_key>(id.id, id.key, id.temp);
+
+  std::string value;
+  auto v = index_->get(key.as_slice(), value);
+  v.ensure();
+  CHECK(v.move_as_ok() == td::KeyValue::GetStatus::Ok);
+
+  auto R = fetch_tl_object<ton_api::db_files_package_value>(value, true);
+  R.ensure();
+  auto x = R.move_as_ok();
+
+  if (x->deleted_) {
+    return;
+  }
+
+  auto &m = get_file_map(id);
+  auto it = m.find(id);
+  if (it == m.end() || it->second.deleted) {
+    return;
+  }
+
+  it->second.deleted = true;
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id](td::Result<td::Unit> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveManager::deleted_package, id);
+  });
+  td::actor::send_closure(it->second.file_actor_id(), &ArchiveSlice::destroy, std::move(P));
+}
+
+void ArchiveManager::deleted_package(PackageId id) {
+  auto key = create_serialize_tl_object<ton_api::db_files_package_key>(id.id, id.key, id.temp);
+
+  std::string value;
+  auto v = index_->get(key.as_slice(), value);
+  v.ensure();
+  CHECK(v.move_as_ok() == td::KeyValue::GetStatus::Ok);
+
+  auto R = fetch_tl_object<ton_api::db_files_package_value>(value, true);
+  R.ensure();
+  auto x = R.move_as_ok();
+
+  if (x->deleted_) {
+    return;
+  }
+  x->deleted_ = true;
+  index_->begin_transaction().ensure();
+  index_->set(key, serialize_tl_object(x, true)).ensure();
+  index_->commit_transaction().ensure();
+
+  auto &m = get_file_map(id);
+  auto it = m.find(id);
+  CHECK(it != m.end());
+  CHECK(it->second.deleted);
+  it->second.clear_actor_id();
+}
+
+void ArchiveManager::load_package(PackageId id) {
+  auto key = create_serialize_tl_object<ton_api::db_files_package_key>(id.id, id.key, id.temp);
+
+  std::string value;
+  auto v = index_->get(key.as_slice(), value);
+  v.ensure();
+  CHECK(v.move_as_ok() == td::KeyValue::GetStatus::Ok);
+
+  auto R = fetch_tl_object<ton_api::db_files_package_value>(value, true);
+  R.ensure();
+  auto x = R.move_as_ok();
+
+  if (x->deleted_) {
+    return;
+  }
+
+  FileDescription desc{id, false};
+  if (!id.temp) {
+    for (auto &e : x->firstblocks_) {
+      desc.first_blocks[ShardIdFull{e->workchain_, static_cast<ShardId>(e->shard_)}] = FileDescription::Desc{
+          static_cast<BlockSeqno>(e->seqno_), static_cast<UnixTime>(e->unixtime_), static_cast<LogicalTime>(e->lt_)};
+    }
+  }
+
+  std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
+  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.key, id.temp, prefix);
+
+  get_file_map(id).emplace(id, std::move(desc));
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno,
+                                                               UnixTime ts, LogicalTime lt, bool force) {
+  auto &f = get_file_map(id);
+  auto it = f.find(id);
+  if (it != f.end()) {
+    if (it->second.deleted) {
+      CHECK(!force);
+      return nullptr;
+    }
+    if (force && !id.temp) {
+      update_desc(it->second, shard, seqno, ts, lt);
+    }
+    return &it->second;
+  }
+  if (!force) {
+    return nullptr;
+  }
+
+  return add_file_desc(shard, id, seqno, ts, lt);
+}
+
+ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno,
+                                                               UnixTime ts, LogicalTime lt) {
+  auto &f = get_file_map(id);
+  CHECK(f.count(id) == 0);
+
+  FileDescription desc{id, false};
+  td::mkdir(db_root_ + id.path()).ensure();
+  std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
+  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.key, id.temp, prefix);
+  if (!id.temp) {
+    update_desc(desc, shard, seqno, ts, lt);
+  }
+
+  std::vector<tl_object_ptr<ton_api::db_files_package_firstBlock>> vec;
+  for (auto &e : desc.first_blocks) {
+    vec.push_back(create_tl_object<ton_api::db_files_package_firstBlock>(e.first.workchain, e.first.shard,
+                                                                         e.second.seqno, e.second.ts, e.second.lt));
+  }
+
+  index_->begin_transaction().ensure();
+  // add package info to list of packages
+  {
+    std::vector<td::int32> t;
+    std::vector<td::int32> tk;
+    std::vector<td::int32> tt;
+    for (auto &e : files_) {
+      t.push_back(e.first.id);
+    }
+    for (auto &e : key_files_) {
+      tk.push_back(e.first.id);
+    }
+    for (auto &e : temp_files_) {
+      tt.push_back(e.first.id);
+    }
+    (id.temp ? tt : (id.key ? tk : t)).push_back(id.id);
+    index_
+        ->set(create_serialize_tl_object<ton_api::db_files_index_key>().as_slice(),
+              create_serialize_tl_object<ton_api::db_files_index_value>(std::move(t), std::move(tk), std::move(tt))
+                  .as_slice())
+        .ensure();
+  }
+  // add package info key
+  {
+    index_
+        ->set(create_serialize_tl_object<ton_api::db_files_package_key>(id.id, id.key, id.temp).as_slice(),
+              create_serialize_tl_object<ton_api::db_files_package_value>(id.id, id.key, id.temp, std::move(vec), false)
+                  .as_slice())
+        .ensure();
+  }
+  index_->commit_transaction().ensure();
+
+  return &f.emplace(id, std::move(desc)).first->second;
+}
+
+void ArchiveManager::update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts,
+                                 LogicalTime lt) {
+  auto it = desc.first_blocks.find(shard);
+  if (it != desc.first_blocks.end() && it->second.seqno <= seqno) {
+    return;
+  }
+  desc.first_blocks[shard] = FileDescription::Desc{seqno, ts, lt};
+  std::vector<tl_object_ptr<ton_api::db_files_package_firstBlock>> vec;
+  for (auto &e : desc.first_blocks) {
+    vec.push_back(create_tl_object<ton_api::db_files_package_firstBlock>(e.first.workchain, e.first.shard,
+                                                                         e.second.seqno, e.second.ts, e.second.lt));
+  }
+  index_->begin_transaction().ensure();
+  index_
+      ->set(create_serialize_tl_object<ton_api::db_files_package_key>(desc.id.id, desc.id.key, desc.id.temp).as_slice(),
+            create_serialize_tl_object<ton_api::db_files_package_value>(desc.id.id, desc.id.key, desc.id.temp,
+                                                                        std::move(vec), false)
+                .as_slice())
+      .ensure();
+  index_->commit_transaction().ensure();
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno,
+                                                                        bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    auto i = it->second.first_blocks.find(shard);
+    if (i != it->second.first_blocks.end() && i->second.seqno <= seqno) {
+      return &it->second;
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts,
+                                                                            bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    auto i = it->second.first_blocks.find(shard);
+    if (i != it->second.first_blocks.end() && i->second.ts <= ts) {
+      return &it->second;
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt,
+                                                                     bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    auto i = it->second.first_blocks.find(shard);
+    if (i != it->second.first_blocks.end() && i->second.lt <= lt) {
+      return &it->second;
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno,
+                                                                        bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  if (account.is_masterchain()) {
+    return get_file_desc_by_seqno(ShardIdFull{masterchainId}, seqno, key_block);
+  }
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    bool found = false;
+    for (int i = 0; i < 60; i++) {
+      auto shard = shard_prefix(account, i);
+      auto it2 = it->second.first_blocks.find(shard);
+      if (it2 != it->second.first_blocks.end()) {
+        if (it2->second.seqno <= seqno) {
+          return &it->second;
+        }
+        found = true;
+      } else if (found) {
+        break;
+      }
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(AccountIdPrefixFull account, UnixTime ts,
+                                                                            bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  if (account.is_masterchain()) {
+    return get_file_desc_by_unix_time(ShardIdFull{masterchainId}, ts, key_block);
+  }
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    bool found = false;
+    for (int i = 0; i < 60; i++) {
+      auto shard = shard_prefix(account, i);
+      auto it2 = it->second.first_blocks.find(shard);
+      if (it2 != it->second.first_blocks.end()) {
+        if (it2->second.ts <= ts) {
+          return &it->second;
+        }
+        found = true;
+      } else if (found) {
+        break;
+      }
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(AccountIdPrefixFull account, LogicalTime lt,
+                                                                     bool key_block) {
+  auto &f = get_file_map(PackageId{0, key_block, false});
+  if (account.is_masterchain()) {
+    return get_file_desc_by_lt(ShardIdFull{masterchainId}, lt, key_block);
+  }
+  for (auto it = f.rbegin(); it != f.rend(); it++) {
+    bool found = false;
+    for (int i = 0; i < 60; i++) {
+      auto shard = shard_prefix(account, i);
+      auto it2 = it->second.first_blocks.find(shard);
+      if (it2 != it->second.first_blocks.end()) {
+        if (it2->second.lt <= lt) {
+          return &it->second;
+        }
+        found = true;
+      } else if (found) {
+        break;
+      }
+    }
+  }
+  return nullptr;
+}
+
+ArchiveManager::FileDescription *ArchiveManager::get_temp_file_desc_by_idx(PackageId idx) {
+  auto it = temp_files_.find(idx);
+  if (it != temp_files_.end()) {
+    if (it->second.deleted) {
+      return nullptr;
+    } else {
+      return &it->second;
+    }
+  } else {
+    return nullptr;
+  }
+}
+
+PackageId ArchiveManager::get_max_temp_file_desc_idx() {
+  if (temp_files_.size() == 0) {
+    return PackageId::empty(false, true);
+  } else {
+    return temp_files_.rbegin()->first;
+  }
+}
+
+PackageId ArchiveManager::get_prev_temp_file_desc_idx(PackageId idx) {
+  auto it = temp_files_.lower_bound(idx);
+  if (it == temp_files_.end()) {
+    return PackageId::empty(false, true);
+  }
+  if (it == temp_files_.begin()) {
+    return PackageId::empty(false, true);
+  }
+  it--;
+  return it->first;
+}
+
+void ArchiveManager::start_up() {
+  td::mkdir(db_root_).ensure();
+  td::mkdir(db_root_ + "/archive/").ensure();
+  td::mkdir(db_root_ + "/archive/tmp/").ensure();
+  td::mkdir(db_root_ + "/archive/packages/").ensure();
+  td::mkdir(db_root_ + "/archive/states/").ensure();
+  td::mkdir(db_root_ + "/files/").ensure();
+  td::mkdir(db_root_ + "/files/packages/").ensure();
+  index_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_root_ + "/files/globalindex").move_as_ok());
+  std::string value;
+  auto v = index_->get(create_serialize_tl_object<ton_api::db_files_index_key>().as_slice(), value);
+  v.ensure();
+  if (v.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+    auto R = fetch_tl_object<ton_api::db_files_index_value>(value, true);
+    R.ensure();
+    auto x = R.move_as_ok();
+
+    for (auto &d : x->packages_) {
+      load_package(PackageId{static_cast<td::uint32>(d), false, false});
+    }
+    for (auto &d : x->key_packages_) {
+      load_package(PackageId{static_cast<td::uint32>(d), true, false});
+    }
+    for (auto &d : x->temp_packages_) {
+      load_package(PackageId{static_cast<td::uint32>(d), false, true});
+    }
+  }
+
+  td::WalkPath::run(db_root_ + "/archive/states/", [&](td::CSlice fname, td::WalkPath::Type t) -> void {
+    if (t == td::WalkPath::Type::NotDir) {
+      LOG(ERROR) << "checking file " << fname;
+      auto pos = fname.rfind('/');
+      if (pos != td::Slice::npos) {
+        fname.remove_prefix(pos + 1);
+      }
+      auto R = FileReferenceShort::create(fname.str());
+      if (R.is_error()) {
+        auto R2 = FileReference::create(fname.str());
+        if (R2.is_error()) {
+          LOG(ERROR) << "deleting bad state file '" << fname << "': " << R.move_as_error() << R2.move_as_error();
+          td::unlink(db_root_ + "/archive/states/" + fname.str()).ignore();
+          return;
+        }
+        auto d = R2.move_as_ok();
+        auto newfname = d.filename_short();
+        td::rename(db_root_ + "/archive/states/" + fname.str(), db_root_ + "/archive/states/" + newfname).ensure();
+        R = FileReferenceShort::create(newfname);
+        R.ensure();
+      }
+      auto f = R.move_as_ok();
+      auto hash = f.hash();
+      perm_states_[hash] = std::move(f);
+    }
+  }).ensure();
+
+  persistent_state_gc(FileHash::zero());
+}
+
+void ArchiveManager::run_gc(UnixTime ts) {
+  auto p = get_temp_package_id_by_unixtime(ts);
+  std::vector<PackageId> vec;
+  for (auto &x : temp_files_) {
+    if (x.first < p) {
+      vec.push_back(x.first);
+    } else {
+      break;
+    }
+  }
+  if (vec.size() <= 1) {
+    return;
+  }
+  vec.resize(vec.size() - 1, PackageId::empty(false, true));
+
+  for (auto &x : vec) {
+    delete_package(x);
+  }
+}
+
+void ArchiveManager::persistent_state_gc(FileHash last) {
+  if (perm_states_.size() == 0) {
+    delay_action(
+        [hash = FileHash::zero(), SelfId = actor_id(this)]() {
+          td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash);
+        },
+        td::Timestamp::in(1.0));
+    return;
+  }
+  auto it = perm_states_.lower_bound(last);
+  if (it != perm_states_.end() && it->first == last) {
+    it++;
+  }
+  if (it == perm_states_.end()) {
+    it = perm_states_.begin();
+  }
+
+  auto &F = it->second;
+  auto hash = F.hash();
+
+  int res = 0;
+  BlockSeqno seqno = 0;
+  F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &x) { res = 1; },
+                               [&](const fileref::PersistentStateShort &x) {
+                                 res = 0;
+                                 seqno = x.masterchain_seqno;
+                               },
+                               [&](const auto &obj) { res = -1; }));
+
+  if (res == -1) {
+    td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore();
+    perm_states_.erase(it);
+  }
+  if (res != 0) {
+    delay_action([hash, SelfId = actor_id(
+                            this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); },
+                 td::Timestamp::in(1.0));
+    return;
+  }
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), hash](td::Result<BlockHandle> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, nullptr, hash);
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, R.move_as_ok(), hash);
+    }
+  });
+
+  get_block_by_seqno(AccountIdPrefixFull{masterchainId, 0}, seqno, std::move(P));
+}
+
+void ArchiveManager::got_gc_masterchain_handle(BlockHandle handle, FileHash hash) {
+  bool to_del = false;
+  if (!handle || !handle->inited_unix_time() || !handle->unix_time()) {
+    to_del = true;
+  } else {
+    auto ttl = ValidatorManager::persistent_state_ttl(handle->unix_time());
+    to_del = ttl < td::Clocks::system();
+  }
+  auto it = perm_states_.find(hash);
+  CHECK(it != perm_states_.end());
+  auto &F = it->second;
+  if (to_del) {
+    td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore();
+    perm_states_.erase(it);
+  }
+  delay_action([hash, SelfId = actor_id(
+                          this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); },
+               td::Timestamp::in(1.0));
+}
+
+PackageId ArchiveManager::get_temp_package_id() const {
+  return get_temp_package_id_by_unixtime(static_cast<UnixTime>(td::Clocks::system()));
+}
+
+PackageId ArchiveManager::get_temp_package_id_by_unixtime(UnixTime ts) const {
+  return PackageId{ts - (ts % 3600), false, true};
+}
+
+PackageId ArchiveManager::get_key_package_id(BlockSeqno seqno) const {
+  return PackageId{seqno - seqno % 200000, true, false};
+}
+
+PackageId ArchiveManager::get_package_id(BlockSeqno seqno) const {
+  auto it = files_.upper_bound(PackageId{seqno, false, false});
+  CHECK(it != files_.begin());
+  it--;
+  return it->first;
+}
+
+PackageId ArchiveManager::get_package_id_force(BlockSeqno masterchain_seqno, ShardIdFull shard, BlockSeqno seqno,
+                                               UnixTime ts, LogicalTime lt, bool is_key) {
+  PackageId p = PackageId::empty(false, false);
+  if (!is_key) {
+    auto it = files_.upper_bound(PackageId{masterchain_seqno, false, false});
+    p = PackageId{masterchain_seqno - (masterchain_seqno % 20000), false, false};
+    if (it != files_.begin()) {
+      it--;
+      if (p < it->first) {
+        p = it->first;
+      }
+    }
+  } else {
+    p = PackageId{masterchain_seqno, false, false};
+  }
+  auto it = files_.find(p);
+  if (it != files_.end()) {
+    return it->first;
+  }
+  add_file_desc(shard, p, seqno, ts, lt);
+  it = files_.find(p);
+  CHECK(it != files_.end());
+  return it->first;
+}
+
+void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) {
+  auto F = get_file_desc_by_seqno(ShardIdFull{masterchainId}, masterchain_seqno, false);
+  if (!F) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found"));
+    return;
+  }
+
+  promise.set_result(F->id.id);
+}
+
+void ArchiveManager::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                                       td::Promise<td::BufferSlice> promise) {
+  if (archive_id != static_cast<td::uint32>(archive_id)) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found"));
+    return;
+  }
+  auto F = get_file_desc(ShardIdFull{masterchainId}, PackageId{static_cast<BlockSeqno>(archive_id), false, false}, 0, 0,
+                         0, false);
+  if (!F) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found"));
+    return;
+  }
+
+  td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_slice, offset, limit, std::move(promise));
+}
+
+void ArchiveManager::commit_transaction() {
+  if (!async_mode_ || huge_transaction_size_++ >= 100) {
+    index_->commit_transaction().ensure();
+    if (async_mode_) {
+      huge_transaction_size_ = 0;
+      huge_transaction_started_ = false;
+    }
+  }
+}
+
+void ArchiveManager::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
+  async_mode_ = mode;
+  if (!async_mode_ && huge_transaction_started_) {
+    index_->commit_transaction().ensure();
+    huge_transaction_size_ = 0;
+    huge_transaction_started_ = false;
+  }
+
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise(std::move(promise));
+
+  for (auto &x : key_files_) {
+    if (!x.second.deleted) {
+      td::actor::send_closure(x.second.file_actor_id(), &ArchiveSlice::set_async_mode, mode, ig.get_promise());
+    }
+  }
+  for (auto &x : temp_files_) {
+    if (!x.second.deleted) {
+      td::actor::send_closure(x.second.file_actor_id(), &ArchiveSlice::set_async_mode, mode, ig.get_promise());
+    }
+  }
+  for (auto &x : files_) {
+    if (!x.second.deleted) {
+      td::actor::send_closure(x.second.file_actor_id(), &ArchiveSlice::set_async_mode, mode, ig.get_promise());
+    }
+  }
+}
+}  // namespace validator
+
+}  // namespace ton

validator/db/archive-manager.hpp

@@ -0,0 +1,155 @@
+#pragma once
+
+#include "archive-slice.hpp"
+
+namespace ton {
+
+namespace validator {
+
+struct PackageId {
+  td::uint32 id;
+  bool key;
+  bool temp;
+
+  explicit PackageId(td::uint32 id, bool key, bool temp) : id(id), key(key), temp(temp) {
+  }
+
+  bool operator<(const PackageId &with) const {
+    return id < with.id;
+  }
+  bool operator==(const PackageId &with) const {
+    return id == with.id;
+  }
+
+  std::string path() const;
+  std::string name() const;
+
+  bool is_empty() {
+    return id == std::numeric_limits<td::uint32>::max();
+  }
+  static PackageId empty(bool key, bool temp) {
+    return PackageId(std::numeric_limits<td::uint32>::max(), key, temp);
+  }
+};
+
+class RootDb;
+
+class ArchiveManager : public td::actor::Actor {
+ public:
+  ArchiveManager(td::actor::ActorId<RootDb> root, std::string db_root);
+
+  void add_handle(BlockHandle handle, td::Promise<td::Unit> promise);
+  void update_handle(BlockHandle handle, td::Promise<td::Unit> promise);
+  void add_file(BlockHandle handle, FileReference ref_id, td::BufferSlice data, td::Promise<td::Unit> promise);
+  void add_key_block_proof(BlockSeqno seqno, UnixTime ts, LogicalTime lt, FileReference ref_id, td::BufferSlice data,
+                           td::Promise<td::Unit> promise);
+  void add_temp_file_short(FileReference ref_id, td::BufferSlice data, td::Promise<td::Unit> promise);
+  void get_handle(BlockIdExt block_id, td::Promise<BlockHandle> promise);
+  void get_key_block_proof(FileReference ref_id, td::Promise<td::BufferSlice> promise);
+  void get_temp_file_short(FileReference ref_id, td::Promise<td::BufferSlice> promise);
+  void get_file_short(FileReference ref_id, td::Promise<td::BufferSlice> promise);
+  void get_file(BlockHandle handle, FileReference ref_id, td::Promise<td::BufferSlice> promise);
+
+  void add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise<td::Unit> promise);
+  void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data,
+                            td::Promise<td::Unit> promise);
+  void get_zero_state(BlockIdExt block_id, td::Promise<td::BufferSlice> promise);
+  void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::BufferSlice> promise);
+  void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,
+                                  td::int64 max_size, td::Promise<td::BufferSlice> promise);
+  void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<bool> promise);
+  void check_zero_state(BlockIdExt block_id, td::Promise<bool> promise);
+
+  void run_gc(UnixTime ts);
+
+  /* from LTDB */
+  void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise<BlockHandle> promise);
+  void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockHandle> promise);
+  void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise<BlockHandle> promise);
+
+  void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise);
+  void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                         td::Promise<td::BufferSlice> promise);
+
+  void start_up() override;
+
+  void begin_transaction();
+  void commit_transaction();
+  void set_async_mode(bool mode, td::Promise<td::Unit> promise);
+
+ private:
+  struct FileDescription {
+    struct Desc {
+      BlockSeqno seqno;
+      UnixTime ts;
+      LogicalTime lt;
+    };
+    FileDescription(PackageId id, bool deleted) : id(id), deleted(deleted) {
+    }
+    auto file_actor_id() const {
+      return file.get();
+    }
+    void clear_actor_id() {
+      file.reset();
+    }
+    PackageId id;
+    bool deleted;
+
+    std::map<ShardIdFull, Desc> first_blocks;
+    td::actor::ActorOwn<ArchiveSlice> file;
+  };
+
+  std::map<PackageId, FileDescription> files_;
+  std::map<PackageId, FileDescription> key_files_;
+  std::map<PackageId, FileDescription> temp_files_;
+  bool async_mode_ = false;
+  bool huge_transaction_started_ = false;
+  td::uint32 huge_transaction_size_ = 0;
+
+  auto &get_file_map(const PackageId &p) {
+    return p.key ? key_files_ : p.temp ? temp_files_ : files_;
+  }
+
+  std::map<FileHash, FileReferenceShort> perm_states_;
+
+  void load_package(PackageId seqno);
+  void delete_package(PackageId seqno);
+  void deleted_package(PackageId seqno);
+  void get_handle_cont(BlockIdExt block_id, PackageId id, td::Promise<BlockHandle> promise);
+  void get_handle_finish(BlockHandle handle, td::Promise<BlockHandle> promise);
+  void get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise<td::BufferSlice> promise);
+
+  FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt,
+                                 bool force);
+  FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt);
+  void update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, LogicalTime lt);
+  FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, bool key_block);
+  FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, bool key_block);
+  FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, bool key_block);
+  FileDescription *get_file_desc_by_seqno(AccountIdPrefixFull shard, BlockSeqno seqno, bool key_block);
+  FileDescription *get_file_desc_by_lt(AccountIdPrefixFull shard, LogicalTime lt, bool key_block);
+  FileDescription *get_file_desc_by_unix_time(AccountIdPrefixFull shard, UnixTime ts, bool key_block);
+  FileDescription *get_temp_file_desc_by_idx(PackageId idx);
+  PackageId get_max_temp_file_desc_idx();
+  PackageId get_prev_temp_file_desc_idx(PackageId id);
+
+  void written_perm_state(FileReferenceShort id);
+
+  void persistent_state_gc(FileHash last);
+  void got_gc_masterchain_handle(BlockHandle handle, FileHash hash);
+
+  std::string db_root_;
+
+  std::shared_ptr<td::KeyValue> index_;
+
+  PackageId get_package_id(BlockSeqno seqno) const;
+  PackageId get_package_id_force(BlockSeqno masterchain_seqno, ShardIdFull shard, BlockSeqno seqno, UnixTime ts,
+                                 LogicalTime lt, bool is_key);
+  PackageId get_temp_package_id() const;
+  PackageId get_key_package_id(BlockSeqno seqno) const;
+  PackageId get_temp_package_id_by_unixtime(UnixTime ts) const;
+};
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/archive-mover.cpp

@@ -0,0 +1,475 @@
+#include "archive-mover.hpp"
+#include "td/actor/MultiPromise.h"
+#include "validator/fabric.h"
+
+namespace ton {
+
+namespace validator {
+
+void ArchiveFileMover::start_up() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+    td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R));
+  });
+  td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, block_id_, std::move(P));
+}
+
+void ArchiveFileMover::got_block_handle0(td::Result<BlockHandle> R) {
+  if (R.is_ok()) {
+    handle_ = R.move_as_ok();
+    CHECK(handle_->moved_to_archive());
+    CHECK(handle_->handle_moved_to_archive());
+    finish_query();
+    return;
+  }
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+    td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R));
+  });
+  td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read_handle, block_id_, std::move(P));
+}
+
+void ArchiveFileMover::got_block_handle1(td::Result<BlockHandle> R) {
+  if (R.is_ok()) {
+    handle_ = R.move_as_ok();
+    got_block_handle();
+    return;
+  }
+
+  if (R.error().code() != ErrorCode::notready) {
+    abort_query(R.move_as_error());
+    return;
+  }
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+    td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R));
+  });
+  td::actor::send_closure(block_db_, &BlockDb::get_block_handle, std::move(P));
+}
+
+void ArchiveFileMover::got_block_handle2(td::Result<BlockHandle> R) {
+  if (R.is_ok()) {
+    handle_ = R.move_as_ok();
+    got_block_handle();
+    return;
+  }
+
+  if (R.error().code() != ErrorCode::notready) {
+    abort_query(R.move_as_error());
+    return;
+  }
+
+  finish_query();
+}
+
+void ArchiveFileMover::got_block_handle() {
+  if (!handle_->is_applied()) {
+    finish_query();
+    return;
+  }
+  if (handle_->id().seqno() == 0) {
+    processed_all_children();
+    return;
+  }
+
+  CHECK(handle_->inited_prev());
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child);
+    }
+  });
+
+  td::actor::create_actor<ArchiveFileMover>("mover", handle_->one_prev(left_), mode_, block_db_, file_db_,
+                                            old_archive_db_, old_archive_manager_, archive_manager_, std::move(P))
+      .release();
+}
+
+void ArchiveFileMover::processed_child() {
+  if (!left_ || !handle_->merge_before()) {
+    processed_all_children();
+    return;
+  }
+  left_ = false;
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child);
+    }
+  });
+
+  td::actor::create_actor<ArchiveFileMover>("mover", handle_->one_prev(left_), mode_, block_db_, file_db_,
+                                            old_archive_db_, old_archive_manager_, archive_manager_, std::move(P))
+      .release();
+}
+
+void ArchiveFileMover::processed_all_children() {
+  if (!handle_->received()) {
+    got_block_data(td::BufferSlice{});
+  } else {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_data, std::move(R));
+    });
+
+    if (handle_->moved_to_archive()) {
+      CHECK(handle_->inited_unix_time());
+      td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(),
+                              handle_->is_key_block(), FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P));
+    } else {
+      td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file,
+                              FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P));
+    }
+  }
+}
+
+void ArchiveFileMover::got_block_data(td::Result<td::BufferSlice> R) {
+  if (R.is_error()) {
+    if (R.error().code() != ErrorCode::notready) {
+      abort_query(R.move_as_error());
+      return;
+    }
+  } else {
+    data_ = R.move_as_ok();
+  }
+  if (!handle_->inited_proof()) {
+    got_block_proof(td::BufferSlice{});
+  } else {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof, std::move(R));
+    });
+
+    if (handle_->moved_to_archive()) {
+      CHECK(handle_->inited_unix_time());
+      td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(),
+                              handle_->is_key_block(), FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P));
+    } else {
+      td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file,
+                              FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P));
+    }
+  }
+}
+
+void ArchiveFileMover::got_block_proof(td::Result<td::BufferSlice> R) {
+  if (R.is_error()) {
+    if (R.error().code() != ErrorCode::notready) {
+      abort_query(R.move_as_error());
+      return;
+    }
+  } else {
+    proof_ = R.move_as_ok();
+  }
+  if (!handle_->inited_proof_link()) {
+    got_block_proof_link(td::BufferSlice{});
+  } else {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+      td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof_link, std::move(R));
+    });
+
+    if (handle_->moved_to_archive()) {
+      CHECK(handle_->inited_unix_time());
+      td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(),
+                              handle_->is_key_block(), FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P));
+    } else {
+      td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file,
+                              FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P));
+    }
+  }
+}
+
+void ArchiveFileMover::got_block_proof_link(td::Result<td::BufferSlice> R) {
+  if (R.is_error()) {
+    if (R.error().code() != ErrorCode::notready) {
+      abort_query(R.move_as_error());
+      return;
+    }
+  } else {
+    proof_link_ = R.move_as_ok();
+  }
+
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveFileMover::written_data);
+  });
+
+  if (data_.size() > 0) {
+    td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Block{block_id_},
+                            std::move(data_), ig.get_promise());
+  }
+  if (proof_.size() > 0) {
+    td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Proof{block_id_},
+                            std::move(proof_), ig.get_promise());
+  }
+  if (proof_link_.size() > 0) {
+    td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::ProofLink{block_id_},
+                            std::move(proof_link_), ig.get_promise());
+  }
+}
+
+void ArchiveFileMover::written_data() {
+  handle_->set_moved_to_archive();
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveFileMover::written_handle);
+  });
+  td::actor::send_closure(archive_manager_, &ArchiveManager::add_handle, handle_, std::move(P));
+}
+
+void ArchiveFileMover::written_handle() {
+  CHECK(handle_->handle_moved_to_archive());
+  finish_query();
+}
+
+void ArchiveFileMover::abort_query(td::Status error) {
+  if (promise_) {
+    promise_.set_error(std::move(error));
+  }
+  stop();
+}
+
+void ArchiveFileMover::finish_query() {
+  if (promise_) {
+    promise_.set_value(td::Unit());
+  }
+  stop();
+}
+
+void ArchiveKeyBlockMover::start_up() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    if (R.is_ok()) {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::skip_block_proof, R.move_as_ok());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof0);
+    }
+  });
+  if (proof_link_) {
+    td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::ProofLink{block_id_},
+                            std::move(P));
+  } else {
+    td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::Proof{block_id_}, std::move(P));
+  }
+}
+
+void ArchiveKeyBlockMover::failed_to_get_proof0() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    if (R.is_ok()) {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof1);
+    }
+  });
+  if (proof_link_) {
+    td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::ProofLink{block_id_},
+                            std::move(P));
+  } else {
+    td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::Proof{block_id_}, std::move(P));
+  }
+}
+
+void ArchiveKeyBlockMover::failed_to_get_proof1() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    if (R.is_ok()) {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof2);
+    }
+  });
+  if (proof_link_) {
+    td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P));
+  } else {
+    td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P));
+  }
+}
+
+void ArchiveKeyBlockMover::failed_to_get_proof2() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    if (R.is_ok()) {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof3);
+    }
+  });
+  if (proof_link_) {
+    td::actor::send_closure(file_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P));
+  } else {
+    td::actor::send_closure(file_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P));
+  }
+}
+
+void ArchiveKeyBlockMover::failed_to_get_proof3() {
+  if (proof_link_) {
+    written_data();
+  } else {
+    proof_link_ = true;
+    start_up();
+  }
+}
+
+void ArchiveKeyBlockMover::got_block_proof(td::BufferSlice data) {
+  data_ = std::move(data);
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::written_data);
+  });
+
+  if (proof_link_) {
+    auto p = create_proof_link(block_id_, data_.clone()).move_as_ok();
+    auto h = p->get_basic_header_info().move_as_ok();
+    td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime,
+                            fileref::ProofLink{block_id_}, std::move(data_), std::move(P));
+  } else {
+    auto p = create_proof(block_id_, data_.clone()).move_as_ok();
+    auto h = p->get_basic_header_info().move_as_ok();
+    td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime, fileref::Proof{block_id_},
+                            std::move(data_), std::move(P));
+  }
+}
+
+void ArchiveKeyBlockMover::skip_block_proof(td::BufferSlice data) {
+  data_ = std::move(data);
+  written_data();
+}
+
+void ArchiveKeyBlockMover::written_data() {
+  td::Ref<ProofLink> proof_link;
+  if (proof_link_) {
+    auto p = create_proof_link(block_id_, data_.clone()).move_as_ok();
+    proof_link = std::move(p);
+  } else {
+    auto p = create_proof(block_id_, data_.clone()).move_as_ok();
+    proof_link = std::move(p);
+  }
+  auto ts = proof_link->get_basic_header_info().move_as_ok().utime;
+  auto te = ValidatorManager::persistent_state_ttl(ts);
+  if (te < td::Clocks::system()) {
+    finish_query();
+    return;
+  }
+}
+
+void ArchiveKeyBlockMover::abort_query(td::Status error) {
+  if (promise_) {
+    promise_.set_error(std::move(error));
+  }
+  stop();
+}
+
+void ArchiveKeyBlockMover::finish_query() {
+  if (promise_) {
+    promise_.set_value(td::Unit());
+  }
+  stop();
+}
+
+void ArchiveMover::start_up() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveMover::moved_blocks);
+    }
+  });
+  td::actor::create_actor<ArchiveFileMover>("fmover", masterchain_block_id_, block_db_.get(), file_db_.get(),
+                                            old_archive_db_.get(), old_archive_manager_.get(), archive_manager_.get(),
+                                            std::move(P))
+      .release();
+}
+
+void ArchiveMover::moved_blocks() {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveMover::got_handle, R.move_as_ok());
+  });
+  td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, masterchain_block_id_, std::move(P));
+}
+
+void ArchiveMover::got_handle(BlockHandle handle) {
+  handle_ = std::move(handle);
+  CHECK(handle_->is_applied());
+  CHECK(handle_->inited_state_boc());
+  CHECK(!handle_->deleted_state_boc());
+  auto P = td::PromiseCreator::lambda(
+      [handle = handle_, SelfId = actor_id(this)](td::Result<td::Ref<vm::DataCell>> R) mutable {
+        R.ensure();
+        auto S = create_shard_state(handle->id(), R.move_as_ok());
+        S.ensure();
+        td::actor::send_closure(SelfId, &ArchiveMover::got_state, td::Ref<MasterchainState>{S.move_as_ok()});
+      });
+  td::actor::send_closure(cell_db_, &CellDb::load_cell, handle_->state(), std::move(P));
+}
+
+void ArchiveMover::got_state(td::Ref<MasterchainState> state) {
+  state_ = std::move(state);
+
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks);
+    }
+  });
+
+  auto k = state_->prev_key_block_id(std::numeric_limits<BlockSeqno>::max());
+  while (k.is_valid() && k.seqno() > 0) {
+    td::actor::create_actor<ArchiveKeyBlockMover>("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(),
+                                                  old_archive_manager_.get(), archive_manager_.get(), ig.get_promise())
+        .release();
+    k = state_->prev_key_block_id(k.seqno());
+  }
+}
+
+void ArchiveMover::moved_key_blocks() {
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+    if (R.is_error()) {
+      td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks);
+    }
+  });
+
+  auto k = state_->prev_key_block_id(std::numeric_limits<BlockSeqno>::max());
+  while (k.is_valid() && k.seqno() > 0) {
+    td::actor::create_actor<ArchiveKeyBlockMover>("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(),
+                                                  old_archive_manager_.get(), archive_manager_.get(), ig.get_promise())
+        .release();
+    k = state_->prev_key_block_id(k.seqno());
+  }
+}
+
+void ArchiveMover::run() {
+  if (to_move_.empty() && to_check_.empty()) {
+    completed();
+    return;
+  }
+  if (!to_check_.empty()) {
+    auto B = to_check_.back();
+    CHECK(to_check_set_.count(B) == 1);
+
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+      td::actor::send_closure(SelfId, &ArchiveMover::got_to_check_handle, std::move(R));
+    });
+    td::actor::send_closure(block_db_, &BlockDb::get_block_handle, B, std::move(P));
+    return;
+  }
+  CHECK(!to_move_.empty());
+}
+
+void ArchiveMover::got_to_check_handle(td::Result<BlockHandle> R) {
+  if (R.is_error()) {
+    CHECK(R.error().code() == ErrorCode::notready);
+    run();
+    return;
+  }
+  auto handle = R.move_as_ok();
+}
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/archive-mover.hpp

@@ -0,0 +1,148 @@
+#pragma once
+
+#include "td/actor/actor.h"
+#include "filedb.hpp"
+#include "blockdb.hpp"
+#include "statedb.hpp"
+#include "celldb.hpp"
+#include "archive-db.hpp"
+#include "archive-manager.hpp"
+
+#include <list>
+#include <set>
+
+namespace ton {
+
+namespace validator {
+
+class ArchiveFileMover : public td::actor::Actor {
+ public:
+  ArchiveFileMover(BlockIdExt block_id, td::actor::ActorId<BlockDb> block_db, td::actor::ActorId<FileDb> file_db,
+                   td::actor::ActorId<FileDb> old_archive_db, td::actor::ActorId<OldArchiveManager> old_archive_manager,
+                   td::actor::ActorId<ArchiveManager> archive_manager, td::Promise<td::Unit> promise)
+      : block_id_(block_id)
+      , block_db_(block_db)
+      , file_db_(file_db)
+      , old_archive_db_(old_archive_db)
+      , old_archive_manager_(old_archive_manager)
+      , archive_manager_(archive_manager)
+      , promise_(std::move(promise)) {
+  }
+  void start_up() override;
+  void got_block_handle0(td::Result<BlockHandle> R);
+  void got_block_handle1(td::Result<BlockHandle> R);
+  void got_block_handle2(td::Result<BlockHandle> R);
+  void got_block_handle();
+
+  void processed_child();
+  void processed_all_children();
+
+  void got_block_data(td::Result<td::BufferSlice> R);
+  void got_block_proof(td::Result<td::BufferSlice> R);
+  void got_block_proof_link(td::Result<td::BufferSlice> R);
+
+  void written_data();
+  void written_handle();
+
+  void abort_query(td::Status error);
+  void finish_query();
+
+ private:
+  BlockIdExt block_id_;
+  BlockHandle handle_;
+  td::BufferSlice data_;
+  td::BufferSlice proof_;
+  td::BufferSlice proof_link_;
+  bool left_ = true;
+
+  td::actor::ActorId<BlockDb> block_db_;
+  td::actor::ActorId<FileDb> file_db_;
+  td::actor::ActorId<FileDb> old_archive_db_;
+  td::actor::ActorId<OldArchiveManager> old_archive_manager_;
+  td::actor::ActorId<ArchiveManager> archive_manager_;
+
+  td::Promise<td::Unit> promise_;
+};
+
+class ArchiveKeyBlockMover : public td::actor::Actor {
+ public:
+  ArchiveKeyBlockMover(BlockIdExt block_id, td::actor::ActorId<BlockDb> block_db, td::actor::ActorId<FileDb> file_db,
+                       td::actor::ActorId<FileDb> old_archive_db,
+                       td::actor::ActorId<OldArchiveManager> old_archive_manager,
+                       td::actor::ActorId<ArchiveManager> archive_manager, td::Promise<td::Unit> promise)
+      : block_id_(block_id)
+      , block_db_(block_db)
+      , file_db_(file_db)
+      , old_archive_db_(old_archive_db)
+      , old_archive_manager_(old_archive_manager)
+      , archive_manager_(archive_manager)
+      , promise_(std::move(promise)) {
+  }
+
+  void start_up() override;
+  void failed_to_get_proof0();
+  void failed_to_get_proof1();
+  void failed_to_get_proof2();
+  void failed_to_get_proof3();
+  void got_block_proof(td::BufferSlice data);
+  void skip_block_proof(td::BufferSlice data);
+
+  void written_data();
+
+  void abort_query(td::Status error);
+  void finish_query();
+
+ private:
+  BlockIdExt block_id_;
+  td::BufferSlice data_;
+  bool proof_link_ = false;
+
+  td::actor::ActorId<BlockDb> block_db_;
+  td::actor::ActorId<FileDb> file_db_;
+  td::actor::ActorId<FileDb> old_archive_db_;
+  td::actor::ActorId<OldArchiveManager> old_archive_manager_;
+  td::actor::ActorId<ArchiveManager> archive_manager_;
+
+  td::Promise<td::Unit> promise_;
+};
+
+class ArchiveMover : public td::actor::Actor {
+ public:
+  ArchiveMover(std::string db_root, BlockIdExt masterchain_block_id, BlockIdExt shard_block_id,
+               BlockIdExt key_block_id);
+
+  void start_up() override;
+  void moved_blocks();
+  void got_handle(BlockHandle handle);
+  void got_state(td::Ref<MasterchainState> state);
+  void moved_key_blocks();
+  void run();
+  void completed();
+  void add_to_move(BlockIdExt block_id);
+  void add_to_check(BlockIdExt block_id);
+
+  void got_to_check_handle(td::Result<BlockHandle> R);
+
+  void abort_query(td::Status error);
+  void finish_query();
+
+ private:
+  std::string db_root_;
+  BlockHandle handle_;
+  td::Ref<MasterchainState> state_;
+
+  td::actor::ActorOwn<BlockDb> block_db_;
+  td::actor::ActorOwn<FileDb> file_db_;
+  td::actor::ActorOwn<FileDb> old_archive_db_;
+  td::actor::ActorOwn<OldArchiveManager> old_archive_manager_;
+  td::actor::ActorOwn<ArchiveManager> archive_manager_;
+  td::actor::ActorOwn<CellDb> cell_db_;
+
+  BlockIdExt masterchain_block_id_;
+  BlockIdExt shard_block_id_;
+  BlockIdExt key_block_id_;
+};
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/archive-slice.cpp

@@ -0,0 +1,470 @@
+#include "archive-slice.hpp"
+
+#include "td/actor/MultiPromise.h"
+#include "validator/fabric.h"
+#include "td/db/RocksDb.h"
+#include "ton/ton-io.hpp"
+#include "td/utils/port/path.h"
+#include "common/delay.h"
+#include "files-async.hpp"
+
+namespace ton {
+
+namespace validator {
+
+void PackageWriter::append(std::string filename, td::BufferSlice data,
+                           td::Promise<std::pair<td::uint64, td::uint64>> promise) {
+  auto offset = package_->append(std::move(filename), std::move(data), !async_mode_);
+  auto size = package_->size();
+
+  promise.set_value(std::pair<td::uint64, td::uint64>{offset, size});
+}
+
+class PackageReader : public td::actor::Actor {
+ public:
+  PackageReader(std::shared_ptr<Package> package, td::uint64 offset,
+                td::Promise<std::pair<std::string, td::BufferSlice>> promise)
+      : package_(std::move(package)), offset_(offset), promise_(std::move(promise)) {
+  }
+  void start_up() {
+    promise_.set_result(package_->read(offset_));
+  }
+
+ private:
+  std::shared_ptr<Package> package_;
+  td::uint64 offset_;
+  td::Promise<std::pair<std::string, td::BufferSlice>> promise_;
+};
+
+void ArchiveSlice::add_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  if (handle->id().seqno() == 0) {
+    update_handle(std::move(handle), std::move(promise));
+    return;
+  }
+  CHECK(!key_blocks_only_);
+  CHECK(!temp_);
+  CHECK(handle->inited_unix_time());
+  CHECK(handle->inited_logical_time());
+
+  auto key = get_db_key_lt_desc(handle->id().shard_full());
+
+  std::string value;
+  auto R = kv_->get(key.as_slice(), value);
+  R.ensure();
+  tl_object_ptr<ton_api::db_lt_desc_value> v;
+  bool add_shard = false;
+  if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+    auto F = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
+    F.ensure();
+    v = F.move_as_ok();
+  } else {
+    v = create_tl_object<ton_api::db_lt_desc_value>(1, 1, 0, 0, 0);
+    add_shard = true;
+  }
+  if (handle->id().seqno() <= static_cast<td::uint32>(v->last_seqno_) ||
+      handle->logical_time() <= static_cast<LogicalTime>(v->last_lt_) ||
+      handle->unix_time() <= static_cast<UnixTime>(v->last_ts_)) {
+    update_handle(std::move(handle), std::move(promise));
+    return;
+  }
+  auto db_value = create_serialize_tl_object<ton_api::db_lt_el_value>(create_tl_block_id(handle->id()),
+                                                                      handle->logical_time(), handle->unix_time());
+  auto db_key = get_db_key_lt_el(handle->id().shard_full(), v->last_idx_++);
+  auto status_key = create_serialize_tl_object<ton_api::db_lt_status_key>();
+  v->last_seqno_ = handle->id().seqno();
+  v->last_lt_ = handle->logical_time();
+  v->last_ts_ = handle->unix_time();
+
+  td::uint32 idx = 0;
+  if (add_shard) {
+    auto G = kv_->get(status_key.as_slice(), value);
+    G.ensure();
+    if (G.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+      idx = 0;
+    } else {
+      auto F = fetch_tl_object<ton_api::db_lt_status_value>(value, true);
+      F.ensure();
+      auto f = F.move_as_ok();
+      idx = f->total_shards_;
+    }
+  }
+
+  auto version = handle->version();
+
+  begin_transaction();
+  kv_->set(key, serialize_tl_object(v, true)).ensure();
+  kv_->set(db_key, db_value.as_slice()).ensure();
+  if (add_shard) {
+    auto shard_key = create_serialize_tl_object<ton_api::db_lt_shard_key>(idx);
+    auto shard_value =
+        create_serialize_tl_object<ton_api::db_lt_shard_value>(handle->id().id.workchain, handle->id().id.shard);
+    kv_->set(status_key.as_slice(), create_serialize_tl_object<ton_api::db_lt_status_value>(idx + 1)).ensure();
+    kv_->set(shard_key.as_slice(), shard_value.as_slice()).ensure();
+  }
+  kv_->set(get_db_key_block_info(handle->id()), handle->serialize().as_slice()).ensure();
+  commit_transaction();
+
+  handle->flushed_upto(version);
+  handle->set_handle_moved_to_archive();
+
+  if (handle->need_flush()) {
+    update_handle(std::move(handle), std::move(promise));
+  } else {
+    promise.set_value(td::Unit());
+  }
+}
+
+void ArchiveSlice::update_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  if (!handle->need_flush() && (temp_ || handle->handle_moved_to_archive())) {
+    promise.set_value(td::Unit());
+    return;
+  }
+  CHECK(!key_blocks_only_);
+
+  begin_transaction();
+  do {
+    auto version = handle->version();
+    kv_->set(get_db_key_block_info(handle->id()), handle->serialize().as_slice()).ensure();
+    handle->flushed_upto(version);
+  } while (handle->need_flush());
+  commit_transaction();
+  if (!temp_) {
+    handle->set_handle_moved_to_archive();
+  }
+
+  promise.set_value(td::Unit());
+}
+
+void ArchiveSlice::add_file(FileReference ref_id, td::BufferSlice data, td::Promise<td::Unit> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  std::string value;
+  auto R = kv_->get(ref_id.hash().to_hex(), value);
+  R.ensure();
+  if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+    promise.set_value(td::Unit());
+    return;
+  }
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id, promise = std::move(promise)](
+                                          td::Result<std::pair<td::uint64, td::uint64>> R) mutable {
+    if (R.is_error()) {
+      promise.set_error(R.move_as_error());
+      return;
+    }
+    auto v = R.move_as_ok();
+    td::actor::send_closure(SelfId, &ArchiveSlice::add_file_cont, std::move(ref_id), v.first, v.second,
+                            std::move(promise));
+  });
+
+  td::actor::send_closure(writer_, &PackageWriter::append, ref_id.filename(), std::move(data), std::move(P));
+}
+
+void ArchiveSlice::add_file_cont(FileReference ref_id, td::uint64 offset, td::uint64 size,
+                                 td::Promise<td::Unit> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  begin_transaction();
+  kv_->set("status", td::to_string(size)).ensure();
+  kv_->set(ref_id.hash().to_hex(), td::to_string(offset)).ensure();
+  commit_transaction();
+  promise.set_value(td::Unit());
+}
+
+void ArchiveSlice::get_handle(BlockIdExt block_id, td::Promise<BlockHandle> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  CHECK(!key_blocks_only_);
+  std::string value;
+  auto R = kv_->get(get_db_key_block_info(block_id), value);
+  R.ensure();
+  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "handle not in archive slice"));
+    return;
+  }
+  auto E = create_block_handle(td::BufferSlice{value});
+  E.ensure();
+  auto handle = E.move_as_ok();
+  if (!temp_) {
+    handle->set_handle_moved_to_archive();
+  }
+  promise.set_value(std::move(handle));
+}
+
+void ArchiveSlice::get_file(FileReference ref_id, td::Promise<td::BufferSlice> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  std::string value;
+  auto R = kv_->get(ref_id.hash().to_hex(), value);
+  R.ensure();
+  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "file not in archive slice"));
+    return;
+  }
+  auto offset = td::to_integer<td::uint64>(value);
+  auto P = td::PromiseCreator::lambda(
+      [promise = std::move(promise)](td::Result<std::pair<std::string, td::BufferSlice>> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          promise.set_value(std::move(R.move_as_ok().second));
+        }
+      });
+  td::actor::create_actor<PackageReader>("reader", package_, offset, std::move(P)).release();
+}
+
+void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id,
+                                    std::function<td::int32(ton_api::db_lt_desc_value &)> compare_desc,
+                                    std::function<td::int32(ton_api::db_lt_el_value &)> compare, bool exact,
+                                    td::Promise<BlockHandle> promise) {
+  if (destroyed_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
+    return;
+  }
+  bool f = false;
+  BlockIdExt block_id;
+  td::uint32 ls = 0;
+  for (td::uint32 len = 0; len <= 60; len++) {
+    auto s = shard_prefix(account_id, len);
+    auto key = get_db_key_lt_desc(s);
+    std::string value;
+    auto F = kv_->get(key, value);
+    F.ensure();
+    if (F.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+      if (!f) {
+        continue;
+      } else {
+        break;
+      }
+    }
+    f = true;
+    auto G = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
+    G.ensure();
+    auto g = G.move_as_ok();
+    if (compare_desc(*g.get()) > 0) {
+      continue;
+    }
+    td::uint32 l = g->first_idx_ - 1;
+    BlockIdExt lseq;
+    td::uint32 r = g->last_idx_;
+    BlockIdExt rseq;
+    while (r - l > 1) {
+      auto x = (r + l) / 2;
+      auto db_key = get_db_key_lt_el(s, x);
+      F = kv_->get(db_key, value);
+      F.ensure();
+      CHECK(F.move_as_ok() == td::KeyValue::GetStatus::Ok);
+      auto E = fetch_tl_object<ton_api::db_lt_el_value>(td::BufferSlice{value}, true);
+      E.ensure();
+      auto e = E.move_as_ok();
+      int cmp_val = compare(*e.get());
+
+      if (cmp_val < 0) {
+        rseq = create_block_id(e->id_);
+        r = x;
+      } else if (cmp_val > 0) {
+        lseq = create_block_id(e->id_);
+        l = x;
+      } else {
+        get_handle(create_block_id(e->id_), std::move(promise));
+        return;
+      }
+    }
+    if (rseq.is_valid()) {
+      if (!block_id.is_valid()) {
+        block_id = rseq;
+      } else if (block_id.id.seqno > rseq.id.seqno) {
+        block_id = rseq;
+      }
+    }
+    if (lseq.is_valid()) {
+      if (ls < lseq.id.seqno) {
+        ls = lseq.id.seqno;
+      }
+    }
+    if (block_id.is_valid() && ls + 1 == block_id.id.seqno) {
+      if (!exact) {
+        get_handle(block_id, std::move(promise));
+      } else {
+        promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found"));
+      }
+      return;
+    }
+  }
+  if (!exact && block_id.is_valid()) {
+    get_handle(block_id, std::move(promise));
+  } else {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found"));
+  }
+}
+
+void ArchiveSlice::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockHandle> promise) {
+  return get_block_common(
+      account_id,
+      [lt](ton_api::db_lt_desc_value &w) {
+        return lt > static_cast<LogicalTime>(w.last_lt_) ? 1 : lt == static_cast<LogicalTime>(w.last_lt_) ? 0 : -1;
+      },
+      [lt](ton_api::db_lt_el_value &w) {
+        return lt > static_cast<LogicalTime>(w.lt_) ? 1 : lt == static_cast<LogicalTime>(w.lt_) ? 0 : -1;
+      },
+      false, std::move(promise));
+}
+
+void ArchiveSlice::get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno,
+                                      td::Promise<BlockHandle> promise) {
+  return get_block_common(
+      account_id,
+      [seqno](ton_api::db_lt_desc_value &w) {
+        return seqno > static_cast<BlockSeqno>(w.last_seqno_)
+                   ? 1
+                   : seqno == static_cast<BlockSeqno>(w.last_seqno_) ? 0 : -1;
+      },
+      [seqno](ton_api::db_lt_el_value &w) {
+        return seqno > static_cast<BlockSeqno>(w.id_->seqno_)
+                   ? 1
+                   : seqno == static_cast<BlockSeqno>(w.id_->seqno_) ? 0 : -1;
+      },
+      true, std::move(promise));
+}
+
+void ArchiveSlice::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts,
+                                          td::Promise<BlockHandle> promise) {
+  return get_block_common(
+      account_id,
+      [ts](ton_api::db_lt_desc_value &w) {
+        return ts > static_cast<UnixTime>(w.last_ts_) ? 1 : ts == static_cast<UnixTime>(w.last_ts_) ? 0 : -1;
+      },
+      [ts](ton_api::db_lt_el_value &w) {
+        return ts > static_cast<UnixTime>(w.ts_) ? 1 : ts == static_cast<UnixTime>(w.ts_) ? 0 : -1;
+      },
+      false, std::move(promise));
+}
+
+td::BufferSlice ArchiveSlice::get_db_key_lt_desc(ShardIdFull shard) {
+  return create_serialize_tl_object<ton_api::db_lt_desc_key>(shard.workchain, shard.shard);
+}
+
+td::BufferSlice ArchiveSlice::get_db_key_lt_el(ShardIdFull shard, td::uint32 idx) {
+  return create_serialize_tl_object<ton_api::db_lt_el_key>(shard.workchain, shard.shard, idx);
+}
+
+td::BufferSlice ArchiveSlice::get_db_key_block_info(BlockIdExt block_id) {
+  return create_serialize_tl_object<ton_api::db_blockdb_key_value>(create_tl_block_id(block_id));
+}
+
+void ArchiveSlice::get_slice(td::uint64 offset, td::uint32 limit, td::Promise<td::BufferSlice> promise) {
+  td::actor::create_actor<db::ReadFile>("readfile", prefix_ + ".pack", offset, limit, 0, std::move(promise)).release();
+}
+
+void ArchiveSlice::start_up() {
+  auto R = Package::open(prefix_ + ".pack", false, true);
+  if (R.is_error()) {
+    LOG(FATAL) << "failed to open/create archive '" << prefix_ << ".pack"
+               << "': " << R.move_as_error();
+    return;
+  }
+  package_ = std::make_shared<Package>(R.move_as_ok());
+  kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(prefix_ + ".index").move_as_ok());
+
+  std::string value;
+  auto R2 = kv_->get("status", value);
+  R2.ensure();
+
+  if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+    auto len = td::to_integer<td::uint64>(value);
+    package_->truncate(len);
+  } else {
+    package_->truncate(0);
+  }
+
+  writer_ = td::actor::create_actor<PackageWriter>("writer", package_);
+}
+
+void ArchiveSlice::begin_transaction() {
+  if (!async_mode_ || !huge_transaction_started_) {
+    kv_->begin_transaction().ensure();
+    if (async_mode_) {
+      huge_transaction_started_ = true;
+    }
+  }
+}
+
+void ArchiveSlice::commit_transaction() {
+  if (!async_mode_ || huge_transaction_size_++ >= 100) {
+    kv_->commit_transaction().ensure();
+    if (async_mode_) {
+      huge_transaction_size_ = 0;
+      huge_transaction_started_ = false;
+    }
+  }
+}
+
+void ArchiveSlice::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
+  async_mode_ = mode;
+  if (!async_mode_ && huge_transaction_started_) {
+    kv_->commit_transaction().ensure();
+    huge_transaction_size_ = 0;
+    huge_transaction_started_ = false;
+  }
+
+  td::actor::send_closure(writer_, &PackageWriter::set_async_mode, mode, std::move(promise));
+}
+
+ArchiveSlice::ArchiveSlice(bool key_blocks_only, bool temp, std::string prefix)
+    : key_blocks_only_(key_blocks_only), temp_(temp), prefix_(std::move(prefix)) {
+}
+
+namespace {
+
+void destroy_db(std::string name, td::uint32 attempt, td::Promise<td::Unit> promise) {
+  auto S = td::RocksDb::destroy(name);
+  if (S.is_ok()) {
+    promise.set_value(td::Unit());
+    return;
+  }
+  if (S.is_error() && attempt > 0 && attempt % 64 == 0) {
+    LOG(ERROR) << "failed to destroy index " << name << ": " << S;
+  } else {
+    LOG(DEBUG) << "failed to destroy index " << name << ": " << S;
+  }
+  delay_action(
+      [name, attempt, promise = std::move(promise)]() mutable { destroy_db(name, attempt, std::move(promise)); },
+      td::Timestamp::in(1.0));
+}
+}  // namespace
+
+void ArchiveSlice::destroy(td::Promise<td::Unit> promise) {
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise(std::move(promise));
+  destroyed_ = true;
+
+  writer_.reset();
+  package_ = nullptr;
+  kv_ = nullptr;
+
+  td::unlink(prefix_ + ".pack").ensure();
+
+  delay_action([name = prefix_ + ".index", attempt = 0,
+                promise = ig.get_promise()]() mutable { destroy_db(name, attempt, std::move(promise)); },
+               td::Timestamp::in(0.0));
+}
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/archive-slice.hpp

@@ -0,0 +1,84 @@
+#pragma once
+
+#include "validator/interfaces/db.h"
+#include "package.hpp"
+#include "fileref.hpp"
+
+namespace ton {
+
+namespace validator {
+
+class PackageWriter : public td::actor::Actor {
+ public:
+  PackageWriter(std::shared_ptr<Package> package) : package_(std::move(package)) {
+  }
+
+  void append(std::string filename, td::BufferSlice data, td::Promise<std::pair<td::uint64, td::uint64>> promise);
+  void set_async_mode(bool mode, td::Promise<td::Unit> promise) {
+    async_mode_ = mode;
+    if (!async_mode_) {
+      package_->sync();
+    }
+    promise.set_value(td::Unit());
+  }
+
+ private:
+  std::shared_ptr<Package> package_;
+  bool async_mode_ = false;
+};
+
+class ArchiveSlice : public td::actor::Actor {
+ public:
+  ArchiveSlice(bool key_blocks_only, bool temp, std::string prefix);
+
+  void add_handle(BlockHandle handle, td::Promise<td::Unit> promise);
+  void update_handle(BlockHandle handle, td::Promise<td::Unit> promise);
+  void add_file(FileReference ref_id, td::BufferSlice data, td::Promise<td::Unit> promise);
+  void get_handle(BlockIdExt block_id, td::Promise<BlockHandle> promise);
+  void get_file(FileReference ref_id, td::Promise<td::BufferSlice> promise);
+
+  /* from LTDB */
+  void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise<BlockHandle> promise);
+  void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockHandle> promise);
+  void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise<BlockHandle> promise);
+  void get_block_common(AccountIdPrefixFull account_id,
+                        std::function<td::int32(ton_api::db_lt_desc_value &)> compare_desc,
+                        std::function<td::int32(ton_api::db_lt_el_value &)> compare, bool exact,
+                        td::Promise<BlockHandle> promise);
+
+  void get_slice(td::uint64 offset, td::uint32 limit, td::Promise<td::BufferSlice> promise);
+
+  void start_up() override;
+  void destroy(td::Promise<td::Unit> promise);
+
+  void begin_transaction();
+  void commit_transaction();
+  void set_async_mode(bool mode, td::Promise<td::Unit> promise);
+
+ private:
+  void written_data(BlockHandle handle, td::Promise<td::Unit> promise);
+  void add_file_cont(FileReference ref_id, td::uint64 offset, td::uint64 size, td::Promise<td::Unit> promise);
+
+  /* ltdb */
+  td::BufferSlice get_db_key_lt_desc(ShardIdFull shard);
+  td::BufferSlice get_db_key_lt_el(ShardIdFull shard, td::uint32 idx);
+  td::BufferSlice get_db_key_block_info(BlockIdExt block_id);
+  td::BufferSlice get_lt_from_db(ShardIdFull shard, td::uint32 idx);
+
+  bool key_blocks_only_;
+  bool temp_;
+
+  bool destroyed_ = false;
+  bool async_mode_ = false;
+  bool huge_transaction_started_ = false;
+  td::uint32 huge_transaction_size_ = 0;
+
+  std::string prefix_;
+  std::shared_ptr<Package> package_;
+  std::shared_ptr<td::KeyValue> kv_;
+  td::actor::ActorOwn<PackageWriter> writer_;
+};
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/archiver.cpp

@@ -24,41 +24,30 @@ namespace ton {
 
 namespace validator {
 
-BlockArchiver::BlockArchiver(BlockIdExt block_id, td::actor::ActorId<RootDb> root_db,
-                             td::actor::ActorId<FileDb> file_db, td::actor::ActorId<FileDb> archive_db,
-                             td::actor::ActorId<ArchiveManager> archive, td::Promise<td::Unit> promise)
-    : block_id_(block_id)
-    , root_db_(root_db)
-    , file_db_(file_db)
-    , archive_db_(archive_db)
-    , archive_(archive)
-    , promise_(std::move(promise)) {
+BlockArchiver::BlockArchiver(BlockHandle handle, td::actor::ActorId<ArchiveManager> archive_db,
+                             td::Promise<td::Unit> promise)
+    : handle_(std::move(handle)), archive_(archive_db), promise_(std::move(promise)) {
 }
 
 void BlockArchiver::start_up() {
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
-    R.ensure();
-    td::actor::send_closure(SelfId, &BlockArchiver::got_block_handle, R.move_as_ok());
-  });
-  td::actor::send_closure(root_db_, &RootDb::get_block_handle_external, block_id_, false, std::move(P));
+  if (handle_->handle_moved_to_archive()) {
+    moved_handle();
+  } else {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+      R.ensure();
+      td::actor::send_closure(SelfId, &BlockArchiver::moved_handle);
+    });
+    td::actor::send_closure(archive_, &ArchiveManager::add_handle, handle_, std::move(P));
+  }
 }
 
-void BlockArchiver::got_block_handle(BlockHandle handle) {
-  handle_ = std::move(handle);
+void BlockArchiver::moved_handle() {
+  CHECK(handle_->handle_moved_to_archive());
   if (handle_->moved_to_archive()) {
     finish_query();
     return;
   }
 
-  if (!handle_->is_applied() && !handle_->is_archived() &&
-      (!handle_->inited_is_key_block() || !handle_->is_key_block())) {
-    // no need for this block
-    // probably this block not in final chain
-    // this will eventually delete all associated data
-    written_block_data();
-    return;
-  }
-
   if (!handle_->inited_proof()) {
     written_proof();
     return;
@@ -69,11 +58,7 @@ void BlockArchiver::got_block_handle(BlockHandle handle) {
     td::actor::send_closure(SelfId, &BlockArchiver::got_proof, R.move_as_ok());
   });
 
-  if (handle_->moved_to_storage()) {
-    td::actor::send_closure(archive_db_, &FileDb::load_file, FileDb::RefId{fileref::Proof{block_id_}}, std::move(P));
-  } else {
-    td::actor::send_closure(file_db_, &FileDb::load_file, FileDb::RefId{fileref::Proof{block_id_}}, std::move(P));
-  }
+  td::actor::send_closure(archive_, &ArchiveManager::get_temp_file_short, fileref::Proof{handle_->id()}, std::move(P));
 }
 
 void BlockArchiver::got_proof(td::BufferSlice data) {
@@ -81,8 +66,8 @@ void BlockArchiver::got_proof(td::BufferSlice data) {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::written_proof);
   });
-  td::actor::send_closure(archive_, &ArchiveManager::write, handle_->unix_time(), handle_->is_key_block(),
-                          FileDb::RefId{fileref::Proof{block_id_}}, std::move(data), std::move(P));
+  td::actor::send_closure(archive_, &ArchiveManager::add_file, handle_, fileref::Proof{handle_->id()}, std::move(data),
+                          std::move(P));
 }
 
 void BlockArchiver::written_proof() {
@@ -95,12 +80,9 @@ void BlockArchiver::written_proof() {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::got_proof_link, R.move_as_ok());
   });
-  if (handle_->moved_to_storage()) {
-    td::actor::send_closure(archive_db_, &FileDb::load_file, FileDb::RefId{fileref::ProofLink{block_id_}},
-                            std::move(P));
-  } else {
-    td::actor::send_closure(file_db_, &FileDb::load_file, FileDb::RefId{fileref::ProofLink{block_id_}}, std::move(P));
-  }
+
+  td::actor::send_closure(archive_, &ArchiveManager::get_temp_file_short, fileref::ProofLink{handle_->id()},
+                          std::move(P));
 }
 
 void BlockArchiver::got_proof_link(td::BufferSlice data) {
@@ -108,8 +90,8 @@ void BlockArchiver::got_proof_link(td::BufferSlice data) {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::written_proof_link);
   });
-  td::actor::send_closure(archive_, &ArchiveManager::write, handle_->unix_time(), handle_->is_key_block(),
-                          FileDb::RefId{fileref::ProofLink{block_id_}}, std::move(data), std::move(P));
+  td::actor::send_closure(archive_, &ArchiveManager::add_file, handle_, fileref::ProofLink{handle_->id()},
+                          std::move(data), std::move(P));
 }
 
 void BlockArchiver::written_proof_link() {
@@ -121,11 +103,8 @@ void BlockArchiver::written_proof_link() {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::got_block_data, R.move_as_ok());
   });
-  if (handle_->moved_to_storage()) {
-    td::actor::send_closure(archive_db_, &FileDb::load_file, FileDb::RefId{fileref::Block{block_id_}}, std::move(P));
-  } else {
-    td::actor::send_closure(file_db_, &FileDb::load_file, FileDb::RefId{fileref::Block{block_id_}}, std::move(P));
-  }
+
+  td::actor::send_closure(archive_, &ArchiveManager::get_temp_file_short, fileref::Block{handle_->id()}, std::move(P));
 }
 
 void BlockArchiver::got_block_data(td::BufferSlice data) {
@@ -133,8 +112,8 @@ void BlockArchiver::got_block_data(td::BufferSlice data) {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::written_block_data);
   });
-  td::actor::send_closure(archive_, &ArchiveManager::write, handle_->unix_time(), handle_->is_key_block(),
-                          FileDb::RefId{fileref::Block{block_id_}}, std::move(data), std::move(P));
+  td::actor::send_closure(archive_, &ArchiveManager::add_file, handle_, fileref::Block{handle_->id()}, std::move(data),
+                          std::move(P));
 }
 
 void BlockArchiver::written_block_data() {
@@ -144,7 +123,7 @@ void BlockArchiver::written_block_data() {
     R.ensure();
     td::actor::send_closure(SelfId, &BlockArchiver::finish_query);
   });
-  td::actor::send_closure(root_db_, &RootDb::store_block_handle, handle_, std::move(P));
+  td::actor::send_closure(archive_, &ArchiveManager::update_handle, handle_, std::move(P));
 }
 
 void BlockArchiver::finish_query() {
@@ -156,7 +135,7 @@ void BlockArchiver::finish_query() {
 
 void BlockArchiver::abort_query(td::Status reason) {
   if (promise_) {
-    VLOG(VALIDATOR_WARNING) << "failed to archive block " << block_id_ << ": " << reason;
+    VLOG(VALIDATOR_WARNING) << "failed to archive block " << handle_->id() << ": " << reason;
     promise_.set_error(std::move(reason));
   }
   stop();

validator/db/archiver.hpp

@@ -22,7 +22,7 @@
 #include "td/actor/actor.h"
 #include "validator/interfaces/block-handle.h"
 #include "ton/ton-io.hpp"
-#include "archive-db.hpp"
+#include "archive-manager.hpp"
 
 namespace ton {
 
@@ -33,14 +33,12 @@ class FileDb;
 
 class BlockArchiver : public td::actor::Actor {
  public:
-  BlockArchiver(BlockIdExt block_id, td::actor::ActorId<RootDb> root_db, td::actor::ActorId<FileDb> file_db,
-                td::actor::ActorId<FileDb> archive_db, td::actor::ActorId<ArchiveManager> archive,
-                td::Promise<td::Unit> promise);
+  BlockArchiver(BlockHandle handle, td::actor::ActorId<ArchiveManager> archive_db, td::Promise<td::Unit> promise);
 
   void abort_query(td::Status error);
 
   void start_up() override;
-  void got_block_handle(BlockHandle handle);
+  void moved_handle();
   void got_proof(td::BufferSlice data);
   void written_proof();
   void got_proof_link(td::BufferSlice data);
@@ -50,14 +48,9 @@ class BlockArchiver : public td::actor::Actor {
   void finish_query();
 
  private:
-  BlockIdExt block_id_;
-  td::actor::ActorId<RootDb> root_db_;
-  td::actor::ActorId<FileDb> file_db_;
-  td::actor::ActorId<FileDb> archive_db_;
+  BlockHandle handle_;
   td::actor::ActorId<ArchiveManager> archive_;
   td::Promise<td::Unit> promise_;
-
-  BlockHandle handle_;
 };
 
 }  // namespace validator

validator/db/blockdb.cpp

@@ -1,301 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#include "blockdb.hpp"
-#include "rootdb.hpp"
-#include "validator/fabric.h"
-#include "ton/ton-tl.hpp"
-#include "td/utils/port/path.h"
-#include "files-async.hpp"
-#include "td/db/RocksDb.h"
-
-namespace ton {
-
-namespace validator {
-
-void BlockDb::store_block_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
-  if (!handle->id().is_valid()) {
-    promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid block id"));
-    return;
-  }
-  auto lru_key = get_block_lru_key(handle->id());
-  auto value_key = get_block_value_key(handle->id());
-  while (handle->need_flush()) {
-    auto version = handle->version();
-    auto R = get_block_value(value_key);
-    if (R.is_ok()) {
-      kv_->begin_transaction().ensure();
-      set_block_value(value_key, handle->serialize());
-      kv_->commit_transaction().ensure();
-    } else {
-      CHECK(get_block_lru(lru_key).is_error());
-      auto empty = get_block_lru_empty_key_hash();
-      auto ER = get_block_lru(empty);
-      ER.ensure();
-      auto E = ER.move_as_ok();
-
-      auto PR = get_block_lru(E.prev);
-      PR.ensure();
-      auto P = PR.move_as_ok();
-      CHECK(P.next == empty);
-
-      DbEntry D{handle->id(), E.prev, empty};
-
-      E.prev = lru_key;
-      P.next = lru_key;
-
-      if (P.is_empty()) {
-        E.next = lru_key;
-        P.prev = lru_key;
-      }
-
-      kv_->begin_transaction().ensure();
-      set_block_value(value_key, handle->serialize());
-      set_block_lru(empty, std::move(E));
-      set_block_lru(D.prev, std::move(P));
-      set_block_lru(lru_key, std::move(D));
-      kv_->commit_transaction().ensure();
-    }
-    handle->flushed_upto(version);
-  }
-  promise.set_value(td::Unit());
-}
-
-void BlockDb::get_block_handle(BlockIdExt id, td::Promise<BlockHandle> promise) {
-  if (!id.is_valid()) {
-    promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid block id"));
-    return;
-  }
-  CHECK(id.is_valid());
-  auto key_hash = get_block_value_key(id);
-  auto B = get_block_value(key_hash);
-  if (B.is_error()) {
-    promise.set_error(B.move_as_error());
-    return;
-  }
-  promise.set_result(create_block_handle(B.move_as_ok()));
-}
-
-void BlockDb::start_up() {
-  kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
-  alarm_timestamp() = td::Timestamp::in(0.1);
-
-  auto empty = get_block_lru_empty_key_hash();
-  if (get_block_lru(empty).is_error()) {
-    DbEntry e{BlockIdExt{}, empty, empty};
-    kv_->begin_transaction().ensure();
-    set_block_lru(empty, std::move(e));
-    kv_->commit_transaction().ensure();
-  }
-  last_gc_ = empty;
-}
-
-BlockDb::BlockDb(td::actor::ActorId<RootDb> root_db, std::string db_path) : root_db_(root_db), db_path_(db_path) {
-}
-
-BlockDb::KeyHash BlockDb::get_block_lru_key(BlockIdExt id) {
-  if (!id.is_valid()) {
-    return KeyHash::zero();
-  } else {
-    auto obj = create_tl_object<ton_api::db_blockdb_key_lru>(create_tl_block_id(id));
-    return get_tl_object_sha_bits256(obj);
-  }
-}
-
-BlockDb::KeyHash BlockDb::get_block_value_key(BlockIdExt id) {
-  CHECK(id.is_valid());
-  auto obj = create_tl_object<ton_api::db_blockdb_key_value>(create_tl_block_id(id));
-  return get_tl_object_sha_bits256(obj);
-}
-
-td::Result<BlockDb::DbEntry> BlockDb::get_block_lru(KeyHash key_hash) {
-  std::string value;
-  auto R = kv_->get(key_hash.as_slice(), value);
-  R.ensure();
-  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-    return td::Status::Error(ErrorCode::notready, "not in db");
-  }
-  auto v = fetch_tl_object<ton_api::db_blockdb_lru>(td::BufferSlice{value}, true);
-  v.ensure();
-  return DbEntry{v.move_as_ok()};
-}
-
-td::Result<td::BufferSlice> BlockDb::get_block_value(KeyHash key_hash) {
-  std::string value;
-  auto R = kv_->get(key_hash.as_slice(), value);
-  R.ensure();
-  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-    return td::Status::Error(ErrorCode::notready, "not in db");
-  }
-  return td::BufferSlice{value};
-}
-
-void BlockDb::set_block_lru(KeyHash key_hash, DbEntry e) {
-  kv_->set(key_hash.as_slice(), serialize_tl_object(e.release(), true)).ensure();
-}
-
-void BlockDb::set_block_value(KeyHash key_hash, td::BufferSlice value) {
-  kv_->set(key_hash.as_slice(), std::move(value)).ensure();
-}
-
-void BlockDb::alarm() {
-  auto R = get_block_lru(last_gc_);
-  R.ensure();
-
-  auto N = R.move_as_ok();
-  if (N.is_empty()) {
-    last_gc_ = N.next;
-    alarm_timestamp() = td::Timestamp::in(0.01);
-    return;
-  }
-
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<bool> R) {
-    if (R.is_error()) {
-      td::actor::send_closure(SelfId, &BlockDb::skip_gc);
-    } else {
-      auto value = R.move_as_ok();
-      if (!value) {
-        td::actor::send_closure(SelfId, &BlockDb::skip_gc);
-      } else {
-        td::actor::send_closure(SelfId, &BlockDb::gc);
-      }
-    }
-  });
-  td::actor::send_closure(root_db_, &RootDb::allow_block_gc, N.block_id, std::move(P));
-}
-
-void BlockDb::gc() {
-  auto FR = get_block_lru(last_gc_);
-  FR.ensure();
-  auto F = FR.move_as_ok();
-
-  auto PR = get_block_lru(F.prev);
-  PR.ensure();
-  auto P = PR.move_as_ok();
-  auto NR = get_block_lru(F.next);
-  NR.ensure();
-  auto N = NR.move_as_ok();
-
-  P.next = F.next;
-  N.prev = F.prev;
-  if (P.is_empty() && N.is_empty()) {
-    P.prev = P.next;
-    N.next = N.prev;
-  }
-
-  auto value_key = get_block_value_key(F.block_id);
-
-  kv_->begin_transaction().ensure();
-  kv_->erase(last_gc_.as_slice()).ensure();
-  kv_->erase(value_key.as_slice()).ensure();
-  set_block_lru(F.prev, std::move(P));
-  set_block_lru(F.next, std::move(N));
-  kv_->commit_transaction().ensure();
-  alarm_timestamp() = td::Timestamp::now();
-
-  DCHECK(get_block_lru(last_gc_).is_error());
-  last_gc_ = F.next;
-}
-
-void BlockDb::skip_gc() {
-  auto R = get_block_lru(last_gc_);
-  R.ensure();
-
-  auto N = R.move_as_ok();
-  last_gc_ = N.next;
-  alarm_timestamp() = td::Timestamp::in(0.01);
-}
-
-BlockDb::DbEntry::DbEntry(tl_object_ptr<ton_api::db_blockdb_lru> entry)
-    : block_id(create_block_id(entry->id_)), prev(entry->prev_), next(entry->next_) {
-}
-
-tl_object_ptr<ton_api::db_blockdb_lru> BlockDb::DbEntry::release() {
-  return create_tl_object<ton_api::db_blockdb_lru>(create_tl_block_id(block_id), prev, next);
-}
-
-void BlockDb::truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) {
-  std::map<ShardIdFull, BlockSeqno> max_seqno;
-  max_seqno.emplace(ShardIdFull{masterchainId}, state->get_seqno() + 1);
-
-  auto shards = state->get_shards();
-  auto it = KeyHash::zero();
-  kv_->begin_transaction().ensure();
-  while (true) {
-    auto R = get_block_lru(it);
-    R.ensure();
-    auto v = R.move_as_ok();
-    it = v.next;
-    R = get_block_lru(it);
-    R.ensure();
-    v = R.move_as_ok();
-    if (v.is_empty()) {
-      break;
-    }
-
-    auto s = v.block_id.shard_full();
-    if (!max_seqno.count(s)) {
-      bool found = false;
-      for (auto &shard : shards) {
-        if (shard_intersects(shard->shard(), s)) {
-          found = true;
-          max_seqno.emplace(s, shard->top_block_id().seqno() + 1);
-          break;
-        }
-      }
-      if (!found) {
-        max_seqno.emplace(s, 0);
-      }
-    }
-
-    bool to_delete = v.block_id.seqno() >= max_seqno[s];
-    if (to_delete) {
-      auto key_hash = get_block_value_key(v.block_id);
-      auto B = get_block_value(key_hash);
-      B.ensure();
-      auto handleR = create_block_handle(B.move_as_ok());
-      handleR.ensure();
-      auto handle = handleR.move_as_ok();
-
-      handle->unsafe_clear_applied();
-      handle->unsafe_clear_next();
-
-      if (handle->need_flush()) {
-        set_block_value(key_hash, handle->serialize());
-      }
-    } else if (v.block_id.seqno() + 1 == max_seqno[s]) {
-      auto key_hash = get_block_value_key(v.block_id);
-      auto B = get_block_value(key_hash);
-      B.ensure();
-      auto handleR = create_block_handle(B.move_as_ok());
-      handleR.ensure();
-      auto handle = handleR.move_as_ok();
-
-      handle->unsafe_clear_next();
-
-      if (handle->need_flush()) {
-        set_block_value(key_hash, handle->serialize());
-      }
-    }
-  }
-  kv_->commit_transaction().ensure();
-}
-
-}  // namespace validator
-
-}  // namespace ton

validator/db/blockdb.hpp

@@ -1,88 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#pragma once
-
-#include "td/actor/actor.h"
-#include "ton/ton-types.h"
-#include "validator/interfaces/block-handle.h"
-#include "validator/interfaces/db.h"
-#include "td/db/KeyValueAsync.h"
-
-namespace ton {
-
-namespace validator {
-
-class RootDb;
-
-class BlockDb : public td::actor::Actor {
- public:
-  void store_block_handle(BlockHandle handle, td::Promise<td::Unit> promise);
-  void get_block_handle(BlockIdExt id, td::Promise<BlockHandle> promise);
-
-  void start_up() override;
-  void alarm() override;
-
-  void gc();
-  void skip_gc();
-
-  void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise);
-
-  BlockDb(td::actor::ActorId<RootDb> root_db, std::string db_path);
-
- private:
-  using KeyHash = td::Bits256;
-  struct DbEntry {
-    BlockIdExt block_id;
-    KeyHash prev;
-    KeyHash next;
-
-    DbEntry(tl_object_ptr<ton_api::db_blockdb_lru> entry);
-    DbEntry() {
-    }
-    DbEntry(BlockIdExt block_id, KeyHash prev, KeyHash next) : block_id(block_id), prev(prev), next(next) {
-    }
-    tl_object_ptr<ton_api::db_blockdb_lru> release();
-    bool is_empty() const {
-      return !block_id.is_valid();
-    }
-  };
-  static KeyHash get_block_lru_key(BlockIdExt block_id);
-  static KeyHash get_block_value_key(BlockIdExt block_id);
-  static KeyHash get_block_lru_empty_key_hash() {
-    return KeyHash::zero();
-  }
-
-  td::Result<DbEntry> get_block_lru(KeyHash key);
-  td::Result<td::BufferSlice> get_block_value(KeyHash key);
-
-  void set_block_lru(KeyHash key_hash, DbEntry e);
-  void set_block_value(KeyHash key_hash, td::BufferSlice data);
-
-  std::shared_ptr<td::KeyValue> kv_;
-
-  td::actor::ActorId<RootDb> root_db_;
-
-  std::string db_path_;
-
-  KeyHash last_gc_ = KeyHash::zero();
-};
-
-}  // namespace validator
-
-}  // namespace ton

validator/db/filedb.cpp

@@ -1,400 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#include "filedb.hpp"
-#include "rootdb.hpp"
-#include "td/utils/port/path.h"
-#include "files-async.hpp"
-#include "adnl/utils.hpp"
-#include "tl-utils/tl-utils.hpp"
-#include "td/utils/overloaded.h"
-
-#include "td/db/RocksDb.h"
-
-namespace ton {
-
-namespace validator {
-
-std::string FileDb::get_file_name(const RefId& ref_id, bool create_dirs) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-
-  auto s = ref_id_hash.to_hex();
-  std::string path = root_path_ + "/files/";
-  for (td::uint32 i = 0; i < depth_; i++) {
-    path = path + s[2 * i] + s[2 * i + 1] + "/";
-    if (create_dirs) {
-      td::mkdir(path).ensure();
-    }
-  }
-  return path + s;
-}
-
-void FileDb::store_file(RefId ref_id, td::BufferSlice data, td::Promise<FileHash> promise) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-  auto R = get_block(ref_id_hash);
-  if (R.is_ok()) {
-    auto val = R.move_as_ok();
-    promise.set_result(val.file_hash);
-    return;
-  }
-
-  auto file_hash = sha256_bits256(data.as_slice());
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), file_hash, ref_id = std::move(ref_id),
-                                       promise = std::move(promise)](td::Result<std::string> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      td::actor::send_closure(SelfId, &FileDb::store_file_continue, std::move(ref_id), file_hash, R.move_as_ok(),
-                              std::move(promise));
-    }
-  });
-  td::actor::create_actor<db::WriteFile>("writefile", root_path_ + "/tmp/", "", std::move(data), std::move(P))
-      .release();
-}
-
-void FileDb::store_file_continue(RefId ref_id, FileHash file_hash, std::string res_path,
-                                 td::Promise<FileHash> promise) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-  auto R = get_block(ref_id_hash);
-  if (R.is_ok()) {
-    td::unlink(res_path).ignore();
-    auto val = R.move_as_ok();
-    promise.set_result(val.file_hash);
-    return;
-  }
-
-  auto path = get_file_name(ref_id, true);
-  td::rename(res_path, path).ensure();
-
-  auto empty = get_empty_ref_id_hash();
-  auto ER = get_block(empty);
-  ER.ensure();
-  auto E = ER.move_as_ok();
-
-  auto PR = get_block(E.prev);
-  PR.ensure();
-  auto P = PR.move_as_ok();
-  CHECK(P.next == empty);
-
-  DbEntry D;
-  D.key = std::move(ref_id);
-  D.prev = E.prev;
-  D.next = empty;
-  D.file_hash = file_hash;
-
-  E.prev = ref_id_hash;
-  P.next = ref_id_hash;
-
-  if (P.is_empty()) {
-    E.next = ref_id_hash;
-    P.prev = ref_id_hash;
-  }
-
-  kv_->begin_transaction().ensure();
-  set_block(empty, std::move(E));
-  set_block(D.prev, std::move(P));
-  set_block(ref_id_hash, std::move(D));
-  kv_->commit_transaction().ensure();
-
-  promise.set_value(std::move(file_hash));
-}
-
-void FileDb::load_file(RefId ref_id, td::Promise<td::BufferSlice> promise) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-  auto R = get_block(ref_id_hash);
-  if (R.is_error()) {
-    promise.set_error(R.move_as_error());
-    return;
-  }
-
-  auto v = R.move_as_ok();
-
-  auto P = td::PromiseCreator::lambda(
-      [promise = std::move(promise), file_hash = v.file_hash](td::Result<td::BufferSlice> R) mutable {
-        if (R.is_error()) {
-          promise.set_error(R.move_as_error());
-        } else {
-          auto data = R.move_as_ok();
-          if (file_hash != sha256_bits256(data.as_slice())) {
-            promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "db error: bad file hash"));
-          } else {
-            promise.set_value(std::move(data));
-          }
-        }
-      });
-
-  td::actor::create_actor<db::ReadFile>("readfile", get_file_name(ref_id, false), 0, -1, 0, std::move(P)).release();
-}
-
-void FileDb::load_file_slice(RefId ref_id, td::int64 offset, td::int64 max_size, td::Promise<td::BufferSlice> promise) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-  auto R = get_block(ref_id_hash);
-  if (R.is_error()) {
-    promise.set_error(R.move_as_error());
-    return;
-  }
-
-  auto v = R.move_as_ok();
-
-  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      promise.set_value(R.move_as_ok());
-    }
-  });
-
-  td::actor::create_actor<db::ReadFile>("readfile", get_file_name(ref_id, false), offset, max_size, 0, std::move(P))
-      .release();
-}
-
-void FileDb::check_file(RefId ref_id, td::Promise<bool> promise) {
-  auto ref_id_hash = get_ref_id_hash(ref_id);
-  auto R = get_block(ref_id_hash);
-  if (R.is_error()) {
-    promise.set_result(false);
-    return;
-  }
-  promise.set_result(true);
-}
-
-td::Slice FileDb::get_key(const RefIdHash& ref) {
-  return ref.as_slice();
-}
-
-td::Result<FileDb::DbEntry> FileDb::get_block(const RefIdHash& ref) {
-  std::string value;
-  auto R = kv_->get(get_key(ref), value);
-  R.ensure();
-  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-    return td::Status::Error(ErrorCode::notready, "not in db");
-  }
-  auto val = fetch_tl_object<ton_api::db_filedb_value>(td::BufferSlice{value}, true);
-  val.ensure();
-  return DbEntry{val.move_as_ok()};
-}
-
-void FileDb::set_block(const RefIdHash& ref, DbEntry entry) {
-  DCHECK(ref == get_ref_id_hash(entry.key));
-  kv_->set(get_key(ref), entry.release()).ensure();
-}
-
-FileDb::FileDb(td::actor::ActorId<RootDb> root_db, std::string root_path, td::uint32 depth, bool is_archive)
-    : root_db_(root_db), root_path_(root_path), depth_(depth), is_archive_(is_archive) {
-}
-
-void FileDb::start_up() {
-  td::mkdir(root_path_).ensure();
-  db_path_ = root_path_ + "/db/";
-
-  kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
-  td::mkdir(root_path_ + "/files/").ensure();
-  td::mkdir(root_path_ + "/tmp/").ensure();
-
-  last_gc_ = get_empty_ref_id_hash();
-  alarm_timestamp() = td::Timestamp::in(0.01);
-
-  auto R = get_block(last_gc_);
-  if (R.is_error()) {
-    DbEntry e{get_empty_ref_id(), last_gc_, last_gc_, FileHash::zero()};
-    kv_->begin_transaction().ensure();
-    set_block(last_gc_, std::move(e));
-    kv_->commit_transaction().ensure();
-  }
-}
-
-void FileDb::alarm() {
-  auto R = get_block(last_gc_);
-  R.ensure();
-  auto N = R.move_as_ok();
-  if (N.is_empty()) {
-    last_gc_ = N.next;
-    alarm_timestamp() = td::Timestamp::in(0.01);
-    return;
-  }
-
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<bool> R) mutable {
-    if (R.is_error()) {
-      td::actor::send_closure(SelfId, &FileDb::skip_gc);
-    } else {
-      auto value = R.move_as_ok();
-      if (!value) {
-        td::actor::send_closure(SelfId, &FileDb::skip_gc);
-      } else {
-        td::actor::send_closure(SelfId, &FileDb::gc);
-      }
-    }
-  });
-  td::actor::send_closure(root_db_, &RootDb::allow_gc, std::move(N.key), is_archive_, std::move(P));
-}
-
-void FileDb::gc() {
-  auto FR = get_block(last_gc_);
-  FR.ensure();
-  auto F = FR.move_as_ok();
-
-  auto PR = get_block(F.prev);
-  PR.ensure();
-  auto P = PR.move_as_ok();
-  auto NR = get_block(F.next);
-  NR.ensure();
-  auto N = NR.move_as_ok();
-
-  P.next = F.next;
-  N.prev = F.prev;
-  if (P.is_empty() && N.is_empty()) {
-    P.prev = P.next;
-    N.next = N.prev;
-  }
-
-  auto name = get_file_name(F.key, false);
-  auto S = td::unlink(name);
-  if (S.is_error()) {
-    LOG(WARNING) << "failed to delete " << name;
-  }
-
-  kv_->begin_transaction().ensure();
-  kv_->erase(last_gc_.as_slice()).ensure();
-  set_block(F.prev, std::move(P));
-  set_block(F.next, std::move(N));
-  kv_->commit_transaction().ensure();
-  alarm_timestamp() = td::Timestamp::now();
-
-  DCHECK(get_block(last_gc_).is_error());
-  last_gc_ = F.next;
-}
-
-void FileDb::skip_gc() {
-  auto FR = get_block(last_gc_);
-  FR.ensure();
-  auto F = FR.move_as_ok();
-  last_gc_ = F.next;
-  alarm_timestamp() = td::Timestamp::in(0.01);
-}
-
-td::BufferSlice FileDb::DbEntry::release() {
-  return create_serialize_tl_object<ton_api::db_filedb_value>(get_ref_id_tl(key), prev, next, file_hash);
-}
-
-bool FileDb::DbEntry::is_empty() const {
-  return (key.get_offset() == key.offset<fileref::Empty>());
-}
-
-FileDb::RefIdHash FileDb::get_ref_id_hash(const RefId& ref) {
-  FileHash x;
-  ref.visit([&](const auto& obj) { x = obj.hash(); });
-  return x;
-}
-
-tl_object_ptr<ton_api::db_filedb_Key> FileDb::get_ref_id_tl(const RefId& ref) {
-  tl_object_ptr<ton_api::db_filedb_Key> x;
-  ref.visit([&](const auto& obj) { x = obj.tl(); });
-  return x;
-}
-
-FileDb::RefId FileDb::get_ref_from_tl(const ton_api::db_filedb_Key& from) {
-  RefId ref_id{fileref::Empty{}};
-  ton_api::downcast_call(
-      const_cast<ton_api::db_filedb_Key&>(from),
-      td::overloaded(
-          [&](const ton_api::db_filedb_key_empty& key) { ref_id = fileref::Empty{}; },
-          [&](const ton_api::db_filedb_key_blockFile& key) { ref_id = fileref::Block{create_block_id(key.block_id_)}; },
-          [&](const ton_api::db_filedb_key_zeroStateFile& key) {
-            ref_id = fileref::ZeroState{create_block_id(key.block_id_)};
-          },
-          [&](const ton_api::db_filedb_key_persistentStateFile& key) {
-            ref_id =
-                fileref::PersistentState{create_block_id(key.block_id_), create_block_id(key.masterchain_block_id_)};
-          },
-          [&](const ton_api::db_filedb_key_proof& key) { ref_id = fileref::Proof{create_block_id(key.block_id_)}; },
-          [&](const ton_api::db_filedb_key_proofLink& key) {
-            ref_id = fileref::ProofLink{create_block_id(key.block_id_)};
-          },
-          [&](const ton_api::db_filedb_key_signatures& key) {
-            ref_id = fileref::Signatures{create_block_id(key.block_id_)};
-          },
-          [&](const ton_api::db_filedb_key_candidate& key) {
-            ref_id = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_),
-                                        key.id_->collated_data_file_hash_};
-          },
-          [&](const ton_api::db_filedb_key_blockInfo& key) {
-            ref_id = fileref::BlockInfo{create_block_id(key.block_id_)};
-          }));
-  return ref_id;
-}
-
-FileDb::RefId FileDb::get_empty_ref_id() {
-  return fileref::Empty();
-}
-
-FileDb::RefIdHash FileDb::get_empty_ref_id_hash() {
-  if (empty_.is_zero()) {
-    empty_ = get_ref_id_hash(get_empty_ref_id());
-  }
-  return empty_;
-}
-
-FileDb::DbEntry::DbEntry(tl_object_ptr<ton_api::db_filedb_value> entry)
-    : key(FileDb::get_ref_from_tl(*entry->key_.get()))
-    , prev(entry->prev_)
-    , next(entry->next_)
-    , file_hash(entry->file_hash_) {
-}
-
-void FileDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
-  std::vector<std::pair<std::string, std::string>> rocksdb_stats;
-  auto stats = kv_->stats();
-  if (stats.size() == 0) {
-    promise.set_value(std::move(rocksdb_stats));
-    return;
-  }
-  size_t pos = 0;
-  while (pos < stats.size()) {
-    while (pos < stats.size() &&
-           (stats[pos] == ' ' || stats[pos] == '\n' || stats[pos] == '\r' || stats[pos] == '\t')) {
-      pos++;
-    }
-    auto p = pos;
-    if (pos == stats.size()) {
-      break;
-    }
-    while (stats[pos] != '\n' && stats[pos] != '\r' && stats[pos] != ' ' && stats[pos] != '\t' && pos < stats.size()) {
-      pos++;
-    }
-    auto name = stats.substr(p, pos - p);
-    if (stats[pos] == '\n' || pos == stats.size()) {
-      rocksdb_stats.emplace_back(name, "");
-      continue;
-    }
-    while (pos < stats.size() &&
-           (stats[pos] == ' ' || stats[pos] == '\n' || stats[pos] == '\r' || stats[pos] == '\t')) {
-      pos++;
-    }
-    p = pos;
-    while (stats[pos] != '\n' && stats[pos] != '\r' && pos < stats.size()) {
-      pos++;
-    }
-    auto value = stats.substr(p, pos - p);
-    rocksdb_stats.emplace_back(name, value);
-  }
-  promise.set_value(std::move(rocksdb_stats));
-}
-
-}  // namespace validator
-
-}  // namespace ton

validator/db/filedb.hpp

@@ -1,218 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#pragma once
-
-#include "ton/ton-types.h"
-#include "td/actor/actor.h"
-#include "validator/interfaces/shard.h"
-#include "td/db/KeyValue.h"
-#include "ton/ton-tl.hpp"
-
-namespace ton {
-
-namespace validator {
-
-namespace fileref {
-
-class Empty {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_empty>();
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_empty>();
-  }
-};
-
-class Block {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_blockFile>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_blockFile>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-
-class ZeroState {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_zeroStateFile>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_zeroStateFile>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-
-class PersistentState {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_persistentStateFile>(create_tl_block_id(block_id),
-                                                                        create_tl_block_id(masterchain_block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_persistentStateFile>(create_tl_block_id(block_id),
-                                                                             create_tl_block_id(masterchain_block_id));
-  }
-
-  BlockIdExt block_id;
-  BlockIdExt masterchain_block_id;
-};
-
-class Proof {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_proof>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_proof>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-
-class ProofLink {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_proofLink>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_proofLink>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-
-class Signatures {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_signatures>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_signatures>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-
-class Candidate {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_candidate>(
-        create_tl_object<ton_api::db_candidate_id>(source.tl(), create_tl_block_id(block_id), collated_data_file_hash));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_candidate>(
-        create_tl_object<ton_api::db_candidate_id>(source.tl(), create_tl_block_id(block_id), collated_data_file_hash));
-  }
-
-  PublicKey source;
-  BlockIdExt block_id;
-  FileHash collated_data_file_hash;
-};
-
-class BlockInfo {
- public:
-  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
-    return create_tl_object<ton_api::db_filedb_key_blockInfo>(create_tl_block_id(block_id));
-  }
-  FileHash hash() const {
-    return create_hash_tl_object<ton_api::db_filedb_key_blockInfo>(create_tl_block_id(block_id));
-  }
-
-  BlockIdExt block_id;
-};
-};  // namespace fileref
-
-class RootDb;
-
-class FileDb : public td::actor::Actor {
- public:
-  using RefId =
-      td::Variant<fileref::Empty, fileref::Block, fileref::ZeroState, fileref::PersistentState, fileref::Proof,
-                  fileref::Proof, fileref::ProofLink, fileref::Signatures, fileref::Candidate, fileref::BlockInfo>;
-  using RefIdHash = td::Bits256;
-
-  void store_file(RefId ref_id, td::BufferSlice data, td::Promise<FileHash> promise);
-  void store_file_continue(RefId ref_id, FileHash file_hash, std::string path, td::Promise<FileHash> promise);
-  void load_file(RefId ref_id, td::Promise<td::BufferSlice> promise);
-  void load_file_slice(RefId ref_id, td::int64 offset, td::int64 max_size, td::Promise<td::BufferSlice> promise);
-  void check_file(RefId ref_id, td::Promise<bool> promise);
-
-  void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise);
-
-  void start_up() override;
-  void alarm() override;
-
-  void gc();
-  void skip_gc();
-
-  FileDb(td::actor::ActorId<RootDb> root_db, std::string root_path, td::uint32 depth, bool is_archive);
-
- private:
-  struct DbEntry {
-    RefId key;
-    RefIdHash prev;
-    RefIdHash next;
-    FileHash file_hash;
-
-    DbEntry(tl_object_ptr<ton_api::db_filedb_value> entry);
-    DbEntry() {
-    }
-    DbEntry(RefId key, RefIdHash prev, RefIdHash next, FileHash file_hash)
-        : key(std::move(key)), prev(prev), next(next), file_hash(file_hash) {
-    }
-    td::BufferSlice release();
-    bool is_empty() const;
-  };
-
-  static RefIdHash get_ref_id_hash(const RefId& ref);
-  static tl_object_ptr<ton_api::db_filedb_Key> get_ref_id_tl(const RefId& ref);
-  static RefId get_ref_from_tl(const ton_api::db_filedb_Key& from);
-  static RefId get_empty_ref_id();
-  RefIdHash get_empty_ref_id_hash();
-
-  std::string get_file_name(const RefId& ref, bool create_dirs);
-  td::Slice get_key(const RefIdHash& ref);
-
-  td::Result<DbEntry> get_block(const RefIdHash& ref_id);
-  void set_block(const RefIdHash& ref_id_hash, DbEntry entry);
-
-  td::actor::ActorId<RootDb> root_db_;
-
-  std::string root_path_;
-  std::string db_path_;
-  td::uint32 depth_;
-
-  bool is_archive_;
-
-  std::shared_ptr<td::KeyValue> kv_;
-
-  RefIdHash last_gc_;
-  RefIdHash empty_ = RefIdHash::zero();
-};
-
-}  // namespace validator
-
-}  // namespace ton
-

validator/db/fileref.cpp

@@ -0,0 +1,471 @@
+#include "fileref.hpp"
+#include "auto/tl/ton_api.hpp"
+#include "td/utils/overloaded.h"
+#include "td/utils/misc.h"
+
+namespace ton {
+
+namespace validator {
+
+namespace {
+
+td::Result<BlockId> get_block_id(std::stringstream& ss) {
+  std::string token;
+  BlockId block_id;
+  std::getline(ss, token, '_');
+  TRY_RESULT(w, td::to_integer_safe<WorkchainId>(token));
+  std::getline(ss, token, '_');
+  TRY_RESULT(shard, td::hex_to_integer_safe<ShardId>(token));
+  std::getline(ss, token, '_');
+  TRY_RESULT(s, td::to_integer_safe<BlockSeqno>(token));
+  return BlockId{w, shard, s};
+}
+
+td::Result<FileHash> get_token_hash(std::stringstream& ss) {
+  std::string token;
+  std::getline(ss, token, '_');
+  if (token.size() != 64) {
+    return td::Status::Error(ErrorCode::protoviolation, "hash must have exactly 64 hexdigits");
+  }
+
+  TRY_RESULT(v, td::hex_decode(token));
+
+  FileHash r;
+  r.as_slice().copy_from(v);
+  return r;
+}
+
+}  // namespace
+
+namespace fileref {
+
+std::string Empty::filename() const {
+  return "empty";
+}
+
+Empty Empty::shortref() const {
+  return *this;
+}
+
+std::string Empty::filename_short() const {
+  return "empty";
+}
+
+BlockShort Block::shortref() const {
+  return BlockShort{block_id.id, hash()};
+}
+
+std::string Block::filename() const {
+  return PSTRING() << "block_" << block_id.to_str();
+}
+
+std::string Block::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "block_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string BlockShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "block_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
+}
+
+ZeroStateShort ZeroState::shortref() const {
+  return ZeroStateShort{block_id.id.workchain, hash()};
+}
+
+std::string ZeroState::filename() const {
+  return PSTRING() << "zerostate_" << block_id.to_str();
+}
+
+std::string ZeroState::filename_short() const {
+  return PSTRING() << "zerostate_" << block_id.id.workchain << "_" << hash().to_hex();
+}
+
+std::string ZeroStateShort::filename_short() const {
+  return PSTRING() << "zerostate_" << workchain << "_" << hash().to_hex();
+}
+
+PersistentStateShort PersistentState::shortref() const {
+  return PersistentStateShort{block_id.shard_full(), masterchain_block_id.seqno(), hash()};
+}
+
+std::string PersistentState::filename() const {
+  return PSTRING() << "state_" << masterchain_block_id.to_str() << "_" << block_id.to_str();
+}
+
+std::string PersistentState::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "state_" << masterchain_block_id.seqno() << "_" << block_id.id.workchain << "_" << s << "_"
+                   << hash().to_hex();
+}
+
+std::string PersistentStateShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(shard_id.shard));
+  return PSTRING() << "state_" << masterchain_seqno << "_" << shard_id.workchain << "_" << s << "_" << hash().to_hex();
+}
+
+ProofShort Proof::shortref() const {
+  return ProofShort{block_id.id, hash()};
+}
+
+std::string Proof::filename() const {
+  return PSTRING() << "proof_" << block_id.to_str();
+}
+
+std::string Proof::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "proof_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string ProofShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "proof_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
+}
+
+ProofLinkShort ProofLink::shortref() const {
+  return ProofLinkShort{block_id.id, hash()};
+}
+
+std::string ProofLink::filename() const {
+  return PSTRING() << "prooflink_" << block_id.to_str();
+}
+
+std::string ProofLink::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "prooflink_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string ProofLinkShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "prooflink_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
+}
+
+SignaturesShort Signatures::shortref() const {
+  return SignaturesShort{block_id.id, hash()};
+}
+
+std::string Signatures::filename() const {
+  return PSTRING() << "signatures_" << block_id.to_str();
+}
+
+std::string Signatures::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "signatures_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string SignaturesShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "signatures_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_"
+                   << hash().to_hex();
+}
+
+CandidateShort Candidate::shortref() const {
+  return CandidateShort{block_id.id, hash()};
+}
+
+std::string Candidate::filename() const {
+  return PSTRING() << "candidate_" << block_id.to_str() << "_" << collated_data_file_hash.to_hex() << "_"
+                   << td::base64url_encode(source.export_as_slice());
+}
+
+std::string Candidate::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "candidate_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string CandidateShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "candidate_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
+}
+
+BlockInfoShort BlockInfo::shortref() const {
+  return BlockInfoShort{block_id.id, hash()};
+}
+
+std::string BlockInfo::filename() const {
+  return PSTRING() << "info_" << block_id.to_str();
+}
+
+std::string BlockInfo::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+  return PSTRING() << "info_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+                   << hash().to_hex();
+}
+
+std::string BlockInfoShort::filename_short() const {
+  char s[33];
+  sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+  return PSTRING() << "info_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
+}
+
+}  // namespace fileref
+
+FileReference::FileReference(tl_object_ptr<ton_api::db_filedb_Key> key) {
+  ton_api::downcast_call(
+      *key.get(),
+      td::overloaded(
+          [&](const ton_api::db_filedb_key_empty& key) { ref_ = fileref::Empty{}; },
+          [&](const ton_api::db_filedb_key_blockFile& key) { ref_ = fileref::Block{create_block_id(key.block_id_)}; },
+          [&](const ton_api::db_filedb_key_zeroStateFile& key) {
+            ref_ = fileref::ZeroState{create_block_id(key.block_id_)};
+          },
+          [&](const ton_api::db_filedb_key_persistentStateFile& key) {
+            ref_ = fileref::PersistentState{create_block_id(key.block_id_), create_block_id(key.masterchain_block_id_)};
+          },
+          [&](const ton_api::db_filedb_key_proof& key) { ref_ = fileref::Proof{create_block_id(key.block_id_)}; },
+          [&](const ton_api::db_filedb_key_proofLink& key) {
+            ref_ = fileref::ProofLink{create_block_id(key.block_id_)};
+          },
+          [&](const ton_api::db_filedb_key_signatures& key) {
+            ref_ = fileref::Signatures{create_block_id(key.block_id_)};
+          },
+          [&](const ton_api::db_filedb_key_candidate& key) {
+            ref_ = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_),
+                                      key.id_->collated_data_file_hash_};
+          },
+          [&](const ton_api::db_filedb_key_blockInfo& key) {
+            ref_ = fileref::BlockInfo{create_block_id(key.block_id_)};
+          }));
+}
+
+FileReferenceShort FileReference::shortref() const {
+  FileReferenceShort h;
+  ref_.visit([&](const auto& obj) { h = obj.shortref(); });
+  return h;
+}
+
+td::Bits256 FileReference::hash() const {
+  FileHash h;
+  ref_.visit([&](const auto& obj) { h = obj.hash(); });
+  return h;
+}
+
+td::Bits256 FileReferenceShort::hash() const {
+  FileHash h;
+  ref_.visit([&](const auto& obj) { h = obj.hash(); });
+  return h;
+}
+
+ShardIdFull FileReference::shard() const {
+  ShardIdFull h;
+  ref_.visit([&](const auto& obj) { h = obj.shard(); });
+  return h;
+}
+
+ShardIdFull FileReferenceShort::shard() const {
+  ShardIdFull h;
+  ref_.visit([&](const auto& obj) { h = obj.shard(); });
+  return h;
+}
+
+std::string FileReference::filename() const {
+  std::string h;
+  ref_.visit([&](const auto& obj) { h = obj.filename(); });
+  return h;
+}
+
+std::string FileReference::filename_short() const {
+  std::string h;
+  ref_.visit([&](const auto& obj) { h = obj.filename_short(); });
+  return h;
+}
+
+std::string FileReferenceShort::filename_short() const {
+  std::string h;
+  ref_.visit([&](const auto& obj) { h = obj.filename_short(); });
+  return h;
+}
+
+td::Result<FileReference> FileReference::create(std::string filename) {
+  std::stringstream ss{filename};
+
+  std::string token;
+  std::getline(ss, token, '_');
+
+  if (token == "empty") {
+    if (ss.eof()) {
+      return fileref::Empty{};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "block") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::Block{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "zerostate") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::ZeroState{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "state") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(masterchain_block_id, BlockIdExt::from_str(token));
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::PersistentState{block_id, masterchain_block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "proof") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::Proof{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "prooflink") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::ProofLink{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "signatures") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::Signatures{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "candidate") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    TRY_RESULT(col_hash, get_token_hash(ss));
+
+    std::string rem = ss.str();
+
+    TRY_RESULT(source_s, td::base64url_decode(rem));
+    TRY_RESULT(source, PublicKey::import(source_s));
+    return fileref::Candidate{source, block_id, col_hash};
+  } else if (token == "info") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(block_id, BlockIdExt::from_str(token));
+    if (ss.eof()) {
+      return fileref::BlockInfo{block_id};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else {
+    return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "unknown prefix '" << token << "'");
+  }
+}
+
+td::Result<FileReferenceShort> FileReferenceShort::create(std::string filename) {
+  std::stringstream ss{filename};
+
+  std::string token;
+  std::getline(ss, token, '_');
+
+  if (token == "empty") {
+    if (ss.eof()) {
+      return fileref::Empty{};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "block") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::BlockShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "zerostate") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(workchain, td::to_integer_safe<WorkchainId>(token));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::ZeroStateShort{workchain, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "state") {
+    std::getline(ss, token, '_');
+    TRY_RESULT(masterchain_seqno, td::to_integer_safe<BlockSeqno>(token));
+    std::getline(ss, token, '_');
+    TRY_RESULT(workchain, td::to_integer_safe<WorkchainId>(token));
+    std::getline(ss, token, '_');
+    TRY_RESULT(shard, td::hex_to_integer_safe<ShardId>(token));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::PersistentStateShort{ShardIdFull{workchain, shard}, masterchain_seqno, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "proof") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::ProofShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "prooflink") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::ProofLinkShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "signatures") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::SignaturesShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "candidate") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::CandidateShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else if (token == "info") {
+    TRY_RESULT(block_id, get_block_id(ss));
+    TRY_RESULT(vhash, get_token_hash(ss));
+    if (ss.eof()) {
+      return fileref::BlockInfoShort{block_id, vhash};
+    } else {
+      return td::Status::Error(ErrorCode::protoviolation, "too big file name");
+    }
+  } else {
+    return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "unknown prefix '" << token << "'");
+  }
+}
+
+}  // namespace validator
+
+}  // namespace ton

validator/db/fileref.hpp

@@ -0,0 +1,372 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "ton/ton-types.h"
+#include "td/actor/actor.h"
+#include "validator/interfaces/shard.h"
+#include "td/db/KeyValue.h"
+#include "ton/ton-tl.hpp"
+
+namespace ton {
+
+namespace validator {
+
+namespace fileref {
+
+class Empty {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_empty>();
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_empty>();
+  }
+  ShardIdFull shard() const {
+    return ShardIdFull{masterchainId};
+  }
+  std::string filename() const;
+  std::string filename_short() const;
+  Empty shortref() const;
+};
+
+class BlockShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class Block {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_blockFile>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_blockFile>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  BlockShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+class ZeroStateShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return ShardIdFull{workchain, shardIdAll};
+  }
+  std::string filename_short() const;
+
+  WorkchainId workchain;
+  FileHash hashv;
+};
+
+class ZeroState {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_zeroStateFile>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_zeroStateFile>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  ZeroStateShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+class PersistentStateShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return shard_id;
+  }
+  std::string filename_short() const;
+
+  ShardIdFull shard_id;
+  BlockSeqno masterchain_seqno;
+  FileHash hashv;
+};
+
+class PersistentState {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_persistentStateFile>(create_tl_block_id(block_id),
+                                                                        create_tl_block_id(masterchain_block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_persistentStateFile>(create_tl_block_id(block_id),
+                                                                             create_tl_block_id(masterchain_block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  PersistentStateShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+  BlockIdExt masterchain_block_id;
+};
+
+class ProofShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class Proof {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_proof>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_proof>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  ProofShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+class ProofLinkShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class ProofLink {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_proofLink>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_proofLink>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  ProofLinkShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+class SignaturesShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class Signatures {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_signatures>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_signatures>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  SignaturesShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+class CandidateShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class Candidate {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_candidate>(
+        create_tl_object<ton_api::db_candidate_id>(source.tl(), create_tl_block_id(block_id), collated_data_file_hash));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_candidate>(
+        create_tl_object<ton_api::db_candidate_id>(source.tl(), create_tl_block_id(block_id), collated_data_file_hash));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  CandidateShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  PublicKey source;
+  BlockIdExt block_id;
+  FileHash collated_data_file_hash;
+};
+
+class BlockInfoShort {
+ public:
+  FileHash hash() const {
+    return hashv;
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  std::string filename_short() const;
+
+  BlockId block_id;
+  FileHash hashv;
+};
+
+class BlockInfo {
+ public:
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+    return create_tl_object<ton_api::db_filedb_key_blockInfo>(create_tl_block_id(block_id));
+  }
+  FileHash hash() const {
+    return create_hash_tl_object<ton_api::db_filedb_key_blockInfo>(create_tl_block_id(block_id));
+  }
+  ShardIdFull shard() const {
+    return block_id.shard_full();
+  }
+  BlockInfoShort shortref() const;
+  std::string filename() const;
+  std::string filename_short() const;
+
+  BlockIdExt block_id;
+};
+
+};  // namespace fileref
+
+class FileReferenceShort {
+ private:
+  td::Variant<fileref::Empty, fileref::BlockShort, fileref::ZeroStateShort, fileref::PersistentStateShort,
+              fileref::ProofShort, fileref::ProofShort, fileref::ProofLinkShort, fileref::SignaturesShort,
+              fileref::CandidateShort, fileref::BlockInfoShort>
+      ref_;
+
+ public:
+  template <typename T>
+  FileReferenceShort(T x) : ref_(std::move(x)) {
+  }
+  FileReferenceShort() : ref_(fileref::Empty{}) {
+  }
+
+  static td::Result<FileReferenceShort> create(std::string filename);
+
+  auto &ref() {
+    return ref_;
+  }
+
+  td::Bits256 hash() const;
+  ShardIdFull shard() const;
+  std::string filename_short() const;
+};
+
+class FileReference {
+ private:
+  td::Variant<fileref::Empty, fileref::Block, fileref::ZeroState, fileref::PersistentState, fileref::Proof,
+              fileref::Proof, fileref::ProofLink, fileref::Signatures, fileref::Candidate, fileref::BlockInfo>
+      ref_;
+
+ public:
+  template <typename T>
+  FileReference(T x) : ref_(std::move(x)) {
+  }
+  FileReference() : ref_(fileref::Empty{}) {
+  }
+  FileReference(tl_object_ptr<ton_api::db_filedb_Key> key);
+
+  static td::Result<FileReference> create(std::string filename);
+
+  auto &ref() {
+    return ref_;
+  }
+
+  FileReferenceShort shortref() const;
+
+  tl_object_ptr<ton_api::db_filedb_Key> tl() const;
+  td::Bits256 hash() const;
+  ShardIdFull shard() const;
+  std::string filename() const;
+  std::string filename_short() const;
+};
+
+}  // namespace validator
+
+}  // namespace ton
+

validator/db/ltdb.cpp

@@ -1,310 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#include "ltdb.hpp"
-#include "rootdb.hpp"
-#include "auto/tl/ton_api.h"
-#include "ton/ton-tl.hpp"
-#include "ton/ton-io.hpp"
-#include "td/db/RocksDb.h"
-
-namespace ton {
-
-namespace validator {
-
-void LtDb::add_new_block(BlockIdExt block_id, LogicalTime lt, UnixTime ts, td::Promise<td::Unit> promise) {
-  auto key = get_desc_key(block_id.shard_full());
-
-  std::string value;
-  auto R = kv_->get(key.as_slice(), value);
-  R.ensure();
-  tl_object_ptr<ton_api::db_lt_desc_value> v;
-  bool add_shard = false;
-  if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
-    auto F = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
-    F.ensure();
-    v = F.move_as_ok();
-  } else {
-    v = create_tl_object<ton_api::db_lt_desc_value>(1, 1, 0, 0, 0);
-    add_shard = true;
-  }
-  if (block_id.id.seqno <= static_cast<td::uint32>(v->last_seqno_) || lt <= static_cast<LogicalTime>(v->last_lt_) ||
-      ts <= static_cast<UnixTime>(v->last_ts_)) {
-    promise.set_value(td::Unit());
-    return;
-  }
-  auto db_value = create_serialize_tl_object<ton_api::db_lt_el_value>(create_tl_block_id(block_id), lt, ts);
-  auto db_key = get_el_key(block_id.shard_full(), v->last_idx_++);
-  auto status_key = create_serialize_tl_object<ton_api::db_lt_status_key>();
-  v->last_seqno_ = block_id.id.seqno;
-  v->last_lt_ = lt;
-  v->last_ts_ = ts;
-
-  td::uint32 idx = 0;
-  if (add_shard) {
-    auto G = kv_->get(status_key.as_slice(), value);
-    G.ensure();
-    if (G.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-      idx = 0;
-    } else {
-      auto F = fetch_tl_object<ton_api::db_lt_status_value>(value, true);
-      F.ensure();
-      auto f = F.move_as_ok();
-      idx = f->total_shards_;
-    }
-  }
-
-  kv_->begin_transaction().ensure();
-  kv_->set(key, serialize_tl_object(v, true)).ensure();
-  kv_->set(db_key, db_value.as_slice()).ensure();
-  if (add_shard) {
-    auto shard_key = create_serialize_tl_object<ton_api::db_lt_shard_key>(idx);
-    auto shard_value = create_serialize_tl_object<ton_api::db_lt_shard_value>(block_id.id.workchain, block_id.id.shard);
-    kv_->set(status_key.as_slice(), create_serialize_tl_object<ton_api::db_lt_status_value>(idx + 1)).ensure();
-    kv_->set(shard_key.as_slice(), shard_value.as_slice()).ensure();
-  }
-  kv_->commit_transaction().ensure();
-
-  promise.set_value(td::Unit());
-}
-
-void LtDb::get_block_common(AccountIdPrefixFull account_id,
-                            std::function<td::int32(ton_api::db_lt_desc_value &)> compare_desc,
-                            std::function<td::int32(ton_api::db_lt_el_value &)> compare, bool exact,
-                            td::Promise<BlockIdExt> promise) {
-  bool f = false;
-  BlockIdExt block_id;
-  td::uint32 ls = 0;
-  for (td::uint32 len = 0; len <= 60; len++) {
-    auto s = shard_prefix(account_id, len);
-    auto key = get_desc_key(s);
-    std::string value;
-    auto F = kv_->get(key, value);
-    F.ensure();
-    if (F.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-      if (!f) {
-        continue;
-      } else {
-        break;
-      }
-    }
-    f = true;
-    auto G = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
-    G.ensure();
-    auto g = G.move_as_ok();
-    if (compare_desc(*g.get()) > 0) {
-      continue;
-    }
-    td::uint32 l = g->first_idx_ - 1;
-    BlockIdExt lseq;
-    td::uint32 r = g->last_idx_;
-    BlockIdExt rseq;
-    while (r - l > 1) {
-      auto x = (r + l) / 2;
-      auto db_key = get_el_key(s, x);
-      F = kv_->get(db_key, value);
-      F.ensure();
-      CHECK(F.move_as_ok() == td::KeyValue::GetStatus::Ok);
-      auto E = fetch_tl_object<ton_api::db_lt_el_value>(td::BufferSlice{value}, true);
-      E.ensure();
-      auto e = E.move_as_ok();
-      int cmp_val = compare(*e.get());
-
-      if (cmp_val < 0) {
-        rseq = create_block_id(e->id_);
-        r = x;
-      } else if (cmp_val > 0) {
-        lseq = create_block_id(e->id_);
-        l = x;
-      } else {
-        promise.set_value(create_block_id(e->id_));
-        return;
-      }
-    }
-    if (rseq.is_valid()) {
-      if (!block_id.is_valid()) {
-        block_id = rseq;
-      } else if (block_id.id.seqno > rseq.id.seqno) {
-        block_id = rseq;
-      }
-    }
-    if (lseq.is_valid()) {
-      if (ls < lseq.id.seqno) {
-        ls = lseq.id.seqno;
-      }
-    }
-    if (block_id.is_valid() && ls + 1 == block_id.id.seqno) {
-      if (!exact) {
-        promise.set_value(std::move(block_id));
-      } else {
-        promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found"));
-      }
-      return;
-    }
-  }
-  if (!exact && block_id.is_valid()) {
-    promise.set_value(std::move(block_id));
-  } else {
-    promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found"));
-  }
-}
-
-void LtDb::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockIdExt> promise) {
-  return get_block_common(
-      account_id,
-      [lt](ton_api::db_lt_desc_value &w) {
-        return lt > static_cast<LogicalTime>(w.last_lt_) ? 1 : lt == static_cast<LogicalTime>(w.last_lt_) ? 0 : -1;
-      },
-      [lt](ton_api::db_lt_el_value &w) {
-        return lt > static_cast<LogicalTime>(w.lt_) ? 1 : lt == static_cast<LogicalTime>(w.lt_) ? 0 : -1;
-      },
-      false, std::move(promise));
-}
-
-void LtDb::get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise<BlockIdExt> promise) {
-  return get_block_common(
-      account_id,
-      [seqno](ton_api::db_lt_desc_value &w) {
-        return seqno > static_cast<BlockSeqno>(w.last_seqno_)
-                   ? 1
-                   : seqno == static_cast<BlockSeqno>(w.last_seqno_) ? 0 : -1;
-      },
-      [seqno](ton_api::db_lt_el_value &w) {
-        return seqno > static_cast<BlockSeqno>(w.id_->seqno_)
-                   ? 1
-                   : seqno == static_cast<BlockSeqno>(w.id_->seqno_) ? 0 : -1;
-      },
-      true, std::move(promise));
-}
-
-void LtDb::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise<BlockIdExt> promise) {
-  return get_block_common(
-      account_id,
-      [ts](ton_api::db_lt_desc_value &w) {
-        return ts > static_cast<UnixTime>(w.last_ts_) ? 1 : ts == static_cast<UnixTime>(w.last_ts_) ? 0 : -1;
-      },
-      [ts](ton_api::db_lt_el_value &w) {
-        return ts > static_cast<UnixTime>(w.ts_) ? 1 : ts == static_cast<UnixTime>(w.ts_) ? 0 : -1;
-      },
-      false, std::move(promise));
-}
-
-td::BufferSlice LtDb::get_desc_key(ShardIdFull shard) {
-  return create_serialize_tl_object<ton_api::db_lt_desc_key>(shard.workchain, shard.shard);
-}
-
-td::BufferSlice LtDb::get_el_key(ShardIdFull shard, td::uint32 idx) {
-  return create_serialize_tl_object<ton_api::db_lt_el_key>(shard.workchain, shard.shard, idx);
-}
-
-void LtDb::start_up() {
-  kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
-}
-
-void LtDb::truncate_workchain(ShardIdFull shard, td::Ref<MasterchainState> state) {
-  auto key = get_desc_key(shard);
-  std::string value;
-  auto R = kv_->get(key, value);
-  R.ensure();
-  CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok);
-  auto F = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
-  F.ensure();
-  auto f = F.move_as_ok();
-
-  auto shards = state->get_shards();
-  BlockSeqno seqno = 0;
-  if (shard.is_masterchain()) {
-    seqno = state->get_seqno();
-  } else {
-    for (auto s : shards) {
-      if (shard_intersects(s->shard(), shard)) {
-        seqno = s->top_block_id().seqno();
-        break;
-      }
-    }
-  }
-
-  while (f->last_idx_ > f->first_idx_) {
-    auto db_key = get_el_key(shard, f->last_idx_ - 1);
-    R = kv_->get(db_key, value);
-    R.ensure();
-    CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok);
-    auto E = fetch_tl_object<ton_api::db_lt_el_value>(td::BufferSlice{value}, true);
-    E.ensure();
-    auto e = E.move_as_ok();
-
-    bool to_delete = static_cast<td::uint32>(e->id_->seqno_) > seqno;
-
-    if (!to_delete) {
-      break;
-    } else {
-      f->last_idx_--;
-      kv_->erase(db_key).ensure();
-    }
-  }
-
-  if (f->first_idx_ == f->last_idx_) {
-    f->last_ts_ = 0;
-    f->last_lt_ = 0;
-    f->last_seqno_ = 0;
-  }
-
-  kv_->set(key, serialize_tl_object(f, true)).ensure();
-}
-
-void LtDb::truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) {
-  auto status_key = create_serialize_tl_object<ton_api::db_lt_status_key>();
-  td::Result<td::KeyValue::GetStatus> R;
-  td::uint32 total_shards = 0;
-  {
-    std::string value;
-    R = kv_->get(status_key.as_slice(), value);
-    R.ensure();
-    if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
-      promise.set_value(td::Unit());
-      return;
-    }
-    auto F = fetch_tl_object<ton_api::db_lt_status_value>(value, true);
-    F.ensure();
-    auto f = F.move_as_ok();
-    total_shards = f->total_shards_;
-    if (total_shards == 0) {
-      promise.set_value(td::Unit());
-      return;
-    }
-  }
-  kv_->begin_transaction().ensure();
-  for (td::uint32 idx = 0; idx < total_shards; idx++) {
-    auto shard_key = create_serialize_tl_object<ton_api::db_lt_shard_key>(idx);
-    std::string value;
-    R = kv_->get(shard_key.as_slice(), value);
-    R.ensure();
-    CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok);
-    auto F = fetch_tl_object<ton_api::db_lt_shard_value>(value, true);
-    F.ensure();
-    auto f = F.move_as_ok();
-
-    truncate_workchain(ShardIdFull{f->workchain_, static_cast<td::uint64>(f->shard_)}, state);
-  }
-  kv_->commit_transaction().ensure();
-  promise.set_value(td::Unit());
-}
-
-}  // namespace validator
-
-}  // namespace ton

validator/db/ltdb.hpp

@@ -1,67 +0,0 @@
-/*
-    This file is part of TON Blockchain Library.
-
-    TON Blockchain Library is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Lesser General Public License as published by
-    the Free Software Foundation, either version 2 of the License, or
-    (at your option) any later version.
-
-    TON Blockchain Library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public License
-    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
-
-    Copyright 2017-2019 Telegram Systems LLP
-*/
-#pragma once
-
-#include "td/actor/actor.h"
-#include "td/db/KeyValueAsync.h"
-#include "validator/interfaces/db.h"
-
-#include "ton/ton-types.h"
-
-#include "auto/tl/ton_api.h"
-
-namespace ton {
-
-namespace validator {
-
-class RootDb;
-
-class LtDb : public td::actor::Actor {
- public:
-  void add_new_block(BlockIdExt block_id, LogicalTime lt, UnixTime ts, td::Promise<td::Unit> promise);
-  void get_block_common(AccountIdPrefixFull account_id,
-                        std::function<td::int32(ton_api::db_lt_desc_value &)> compare_desc,
-                        std::function<td::int32(ton_api::db_lt_el_value &)> compare, bool exact,
-                        td::Promise<BlockIdExt> promise);
-  void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<BlockIdExt> promise);
-  void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise<BlockIdExt> promise);
-  void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise<BlockIdExt> promise);
-
-  void truncate_workchain(ShardIdFull shard, td::Ref<MasterchainState> state);
-  void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise);
-
-  void start_up() override;
-
-  LtDb(td::actor::ActorId<RootDb> root_db, std::string db_path) : root_db_(root_db), db_path_(std::move(db_path)) {
-  }
-
- private:
-  td::BufferSlice get_desc_key(ShardIdFull shard);
-  td::BufferSlice get_el_key(ShardIdFull shard, td::uint32 idx);
-  td::BufferSlice get_from_db(ShardIdFull shard, td::uint32 idx);
-
-  std::shared_ptr<td::KeyValue> kv_;
-
-  td::actor::ActorId<RootDb> root_db_;
-  std::string db_path_;
-};
-
-}  // namespace validator
-
-}  // namespace ton

validator/db/package.cpp

@@ -34,7 +34,7 @@ td::Status Package::truncate(td::uint64 size) {
   return fd_.truncate_to_current_position(size + header_size());
 }
 
-td::uint64 Package::append(std::string filename, td::Slice data) {
+td::uint64 Package::append(std::string filename, td::Slice data, bool sync) {
   CHECK(data.size() <= max_data_size());
   CHECK(filename.size() <= max_filename_size());
   auto size = fd_.get_size().move_as_ok();
@@ -48,10 +48,16 @@ td::uint64 Package::append(std::string filename, td::Slice data) {
   size += filename.size();
   CHECK(fd_.pwrite(data, size).move_as_ok() == data.size());
   size += data.size();
-  fd_.sync().ensure();
+  if (sync) {
+    fd_.sync().ensure();
+  }
   return orig_size - header_size();
 }
 
+void Package::sync() {
+  fd_.sync().ensure();
+}
+
 td::uint64 Package::size() const {
   return fd_.get_size().move_as_ok() - header_size();
 }
@@ -140,4 +146,28 @@ td::Result<Package> Package::open(std::string path, bool read_only, bool create)
   return Package{std::move(fd)};
 }
 
+void Package::iterate(std::function<bool(std::string, td::BufferSlice, td::uint64)> func) {
+  td::uint64 p = 0;
+
+  td::uint64 size = fd_.get_size().move_as_ok();
+  if (size < header_size()) {
+    LOG(ERROR) << "too short archive";
+    return;
+  }
+  size -= header_size();
+  while (p != size) {
+    auto R = read(p);
+    if (R.is_error()) {
+      LOG(ERROR) << "broken archive: " << R.move_as_error();
+      return;
+    }
+    auto q = R.move_as_ok();
+    if (!func(q.first, q.second.clone(), p)) {
+      break;
+    }
+
+    p = advance(p).move_as_ok();
+  }
+}
+
 }  // namespace ton

validator/db/package.hpp

@@ -14,24 +14,17 @@ class Package {
 
   td::Status truncate(td::uint64 size);
 
-  td::uint64 append(std::string filename, td::Slice data);
+  td::uint64 append(std::string filename, td::Slice data, bool sync = true);
+  void sync();
   td::uint64 size() const;
   td::Result<std::pair<std::string, td::BufferSlice>> read(td::uint64 offset) const;
+
   td::Result<td::uint64> advance(td::uint64 offset);
+  void iterate(std::function<bool(std::string, td::BufferSlice, td::uint64)> func);
 
-  struct Iterator {
-    td::uint64 offset;
-    Package &package;
-
-    Iterator operator++(int);
-    const Iterator operator++(int) const;
-    td::Result<std::pair<std::string, td::BufferSlice>> read() const;
-  };
-
-  Iterator begin();
-  const Iterator begin() const;
-  Iterator end();
-  const Iterator end() const;
+  td::FileFd &fd() {
+    return fd_;
+  }
 
  private:
   td::FileFd fd_;

validator/db/rootdb.cpp

@@ -32,22 +32,22 @@ namespace ton {
 namespace validator {
 
 void RootDb::store_block_data(BlockHandle handle, td::Ref<BlockData> block, td::Promise<td::Unit> promise) {
-  if (handle->moved_to_storage() || handle->moved_to_archive()) {
+  if (handle->received()) {
     promise.set_value(td::Unit());
     return;
   }
-  auto id = block_db_.get();
 
-  auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      handle->set_received();
-      td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise));
-    }
-  });
+  auto P = td::PromiseCreator::lambda(
+      [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          handle->set_received();
+          td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise));
+        }
+      });
 
-  td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::Block{handle->id()}}, block->data(),
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_file, handle, fileref::Block{handle->id()}, block->data(),
                           std::move(P));
 }
 
@@ -64,45 +64,34 @@ void RootDb::get_block_data(BlockHandle handle, td::Promise<td::Ref<BlockData>>
           }
         });
 
-    if (handle->moved_to_archive()) {
-      td::actor::send_closure(new_archive_db_, &ArchiveManager::read, handle->unix_time(), handle->is_key_block(),
-                              FileDb::RefId{fileref::Block{handle->id()}}, std::move(P));
-    } else {
-      td::actor::send_closure(handle->moved_to_storage() ? old_archive_db_.get() : file_db_.get(), &FileDb::load_file,
-                              FileDb::RefId{fileref::Block{handle->id()}}, std::move(P));
-    }
+    td::actor::send_closure(archive_db_, &ArchiveManager::get_file, handle, fileref::Block{handle->id()}, std::move(P));
   }
 }
 
 void RootDb::store_block_signatures(BlockHandle handle, td::Ref<BlockSignatureSet> data,
                                     td::Promise<td::Unit> promise) {
-  if (handle->moved_to_storage() || handle->moved_to_archive()) {
+  if (handle->inited_signatures() || handle->moved_to_archive()) {
     promise.set_value(td::Unit());
     return;
   }
-  auto id = block_db_.get();
 
-  auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      handle->set_signatures();
-      td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise));
-    }
-  });
-
-  td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::Signatures{handle->id()}},
+  auto P = td::PromiseCreator::lambda(
+      [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          handle->set_signatures();
+          td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise));
+        }
+      });
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_temp_file_short, fileref::Signatures{handle->id()},
                           data->serialize(), std::move(P));
 }
 
 void RootDb::get_block_signatures(BlockHandle handle, td::Promise<td::Ref<BlockSignatureSet>> promise) {
-  if (!handle->inited_signatures()) {
+  if (!handle->inited_signatures() || handle->moved_to_archive()) {
     promise.set_error(td::Status::Error(ErrorCode::notready, "not in db"));
   } else {
-    if (handle->moved_to_storage() || handle->moved_to_archive()) {
-      promise.set_error(td::Status::Error(ErrorCode::error, "signatures already gc'd"));
-      return;
-    }
     auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
       if (R.is_error()) {
         promise.set_error(R.move_as_error());
@@ -110,28 +99,28 @@ void RootDb::get_block_signatures(BlockHandle handle, td::Promise<td::Ref<BlockS
         promise.set_result(create_signature_set(R.move_as_ok()));
       }
     });
-    td::actor::send_closure(file_db_, &FileDb::load_file, FileDb::RefId{fileref::Signatures{handle->id()}},
+    td::actor::send_closure(archive_db_, &ArchiveManager::get_temp_file_short, fileref::Signatures{handle->id()},
                             std::move(P));
   }
 }
 
 void RootDb::store_block_proof(BlockHandle handle, td::Ref<Proof> proof, td::Promise<td::Unit> promise) {
-  if (handle->moved_to_storage() || handle->moved_to_archive()) {
+  if (handle->inited_proof()) {
     promise.set_value(td::Unit());
     return;
   }
-  auto id = block_db_.get();
 
-  auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      handle->set_proof();
-      td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise));
-    }
-  });
+  auto P = td::PromiseCreator::lambda(
+      [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          handle->set_proof();
+          td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise));
+        }
+      });
 
-  td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::Proof{handle->id()}}, proof->data(),
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_file, handle, fileref::Proof{handle->id()}, proof->data(),
                           std::move(P));
 }
 
@@ -147,34 +136,27 @@ void RootDb::get_block_proof(BlockHandle handle, td::Promise<td::Ref<Proof>> pro
             promise.set_result(create_proof(id, R.move_as_ok()));
           }
         });
-    if (handle->moved_to_archive()) {
-      td::actor::send_closure(new_archive_db_, &ArchiveManager::read, handle->unix_time(), handle->is_key_block(),
-                              FileDb::RefId{fileref::Proof{handle->id()}}, std::move(P));
-    } else {
-      td::actor::send_closure(handle->moved_to_storage() ? old_archive_db_.get() : file_db_.get(), &FileDb::load_file,
-                              FileDb::RefId{fileref::Proof{handle->id()}}, std::move(P));
-    }
+    td::actor::send_closure(archive_db_, &ArchiveManager::get_file, handle, fileref::Proof{handle->id()}, std::move(P));
   }
 }
 
 void RootDb::store_block_proof_link(BlockHandle handle, td::Ref<ProofLink> proof, td::Promise<td::Unit> promise) {
-  if (handle->moved_to_storage() || handle->moved_to_archive()) {
+  if (handle->inited_proof_link()) {
     promise.set_value(td::Unit());
     return;
   }
-  auto id = block_db_.get();
 
-  auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      handle->set_proof_link();
-      td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise));
-    }
-  });
-
-  td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::ProofLink{handle->id()}}, proof->data(),
-                          std::move(P));
+  auto P = td::PromiseCreator::lambda(
+      [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          handle->set_proof_link();
+          td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise));
+        }
+      });
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_file, handle, fileref::ProofLink{handle->id()},
+                          proof->data(), std::move(P));
 }
 
 void RootDb::get_block_proof_link(BlockHandle handle, td::Promise<td::Ref<ProofLink>> promise) {
@@ -189,13 +171,8 @@ void RootDb::get_block_proof_link(BlockHandle handle, td::Promise<td::Ref<ProofL
             promise.set_result(create_proof_link(id, R.move_as_ok()));
           }
         });
-    if (handle->moved_to_archive()) {
-      td::actor::send_closure(new_archive_db_, &ArchiveManager::read, handle->unix_time(), handle->is_key_block(),
-                              FileDb::RefId{fileref::ProofLink{handle->id()}}, std::move(P));
-    } else {
-      td::actor::send_closure(handle->moved_to_storage() ? old_archive_db_.get() : file_db_.get(), &FileDb::load_file,
-                              FileDb::RefId{fileref::ProofLink{handle->id()}}, std::move(P));
-    }
+    td::actor::send_closure(archive_db_, &ArchiveManager::get_file, handle, fileref::ProofLink{handle->id()},
+                            std::move(P));
   }
 }
 
@@ -204,16 +181,16 @@ void RootDb::store_block_candidate(BlockCandidate candidate, td::Promise<td::Uni
       PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}.tl(), create_tl_block_id(candidate.id),
       std::move(candidate.data), std::move(candidate.collated_data));
 
-  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<FileHash> R) mutable {
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Unit> R) mutable {
     if (R.is_error()) {
       promise.set_error(R.move_as_error());
     } else {
       promise.set_value(td::Unit());
     }
   });
-  td::actor::send_closure(file_db_, &FileDb::store_file,
-                          FileDb::RefId{fileref::Candidate{PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}},
-                                                           candidate.id, candidate.collated_file_hash}},
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_temp_file_short,
+                          fileref::Candidate{PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.id,
+                                             candidate.collated_file_hash},
                           std::move(obj), std::move(P));
 }
 
@@ -234,18 +211,18 @@ void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash colla
                                        std::move(val->collated_data_)});
     }
   });
-  td::actor::send_closure(file_db_, &FileDb::load_file,
-                          FileDb::RefId{fileref::Candidate{source, id, collated_data_file_hash}}, std::move(P));
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_temp_file_short,
+                          fileref::Candidate{source, id, collated_data_file_hash}, std::move(P));
 }
 
 void RootDb::store_block_state(BlockHandle handle, td::Ref<ShardState> state,
                                td::Promise<td::Ref<ShardState>> promise) {
-  if (handle->moved_to_storage() || handle->moved_to_archive()) {
+  if (handle->moved_to_archive()) {
     promise.set_value(std::move(state));
     return;
   }
   if (!handle->inited_state_boc()) {
-    auto P = td::PromiseCreator::lambda([b = block_db_.get(), root_hash = state->root_hash(), handle,
+    auto P = td::PromiseCreator::lambda([b = archive_db_.get(), root_hash = state->root_hash(), handle,
                                          promise = std::move(promise)](td::Result<td::Ref<vm::DataCell>> R) mutable {
       if (R.is_error()) {
         promise.set_error(R.move_as_error());
@@ -262,7 +239,7 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref<ShardState> state,
               promise.set_value(std::move(state));
             });
 
-        td::actor::send_closure(b, &BlockDb::store_block_handle, std::move(handle), std::move(P));
+        td::actor::send_closure(b, &ArchiveManager::update_handle, std::move(handle), std::move(P));
       }
     });
     td::actor::send_closure(cell_db_, &CellDb::store_cell, handle->id(), state->root_cell(), std::move(P));
@@ -295,83 +272,46 @@ void RootDb::get_block_state(BlockHandle handle, td::Promise<td::Ref<ShardState>
 
 void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
                                          td::Promise<td::Unit> promise) {
-  auto id = block_db_.get();
-
-  auto P = td::PromiseCreator::lambda([id, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      promise.set_value(td::Unit());
-    }
-  });
-
-  td::actor::send_closure(old_archive_db_, &FileDb::store_file,
-                          FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, std::move(state),
-                          std::move(P));
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id,
+                          std::move(state), std::move(promise));
 }
 
 void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id,
                                        td::Promise<td::BufferSlice> promise) {
-  td::actor::send_closure(old_archive_db_, &FileDb::load_file,
-                          FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, std::move(promise));
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id,
+                          std::move(promise));
 }
 
 void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,
                                              td::int64 max_size, td::Promise<td::BufferSlice> promise) {
-  td::actor::send_closure(old_archive_db_, &FileDb::load_file_slice,
-                          FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, offset, max_size,
-                          std::move(promise));
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_slice, block_id, masterchain_block_id,
+                          offset, max_size, std::move(promise));
 }
 
 void RootDb::check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id,
                                                 td::Promise<bool> promise) {
-  td::actor::send_closure(old_archive_db_, &FileDb::check_file,
-                          FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, std::move(promise));
+  td::actor::send_closure(archive_db_, &ArchiveManager::check_persistent_state, block_id, masterchain_block_id,
+                          std::move(promise));
 }
 
 void RootDb::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise<td::Unit> promise) {
-  auto id = block_db_.get();
-
-  auto P = td::PromiseCreator::lambda([id, promise = std::move(promise)](td::Result<FileHash> R) mutable {
-    if (R.is_error()) {
-      promise.set_error(R.move_as_error());
-    } else {
-      promise.set_value(td::Unit());
-    }
-  });
-
-  td::actor::send_closure(old_archive_db_, &FileDb::store_file, FileDb::RefId{fileref::ZeroState{block_id}},
-                          std::move(state), std::move(P));
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_zero_state, block_id, std::move(state), std::move(promise));
 }
 
 void RootDb::get_zero_state_file(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {
-  td::actor::send_closure(old_archive_db_, &FileDb::load_file, FileDb::RefId{fileref::ZeroState{block_id}},
-                          std::move(promise));
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_zero_state, block_id, std::move(promise));
 }
 
 void RootDb::check_zero_state_file_exists(BlockIdExt block_id, td::Promise<bool> promise) {
-  td::actor::send_closure(old_archive_db_, &FileDb::check_file, FileDb::RefId{fileref::ZeroState{block_id}},
-                          std::move(promise));
+  td::actor::send_closure(archive_db_, &ArchiveManager::check_zero_state, block_id, std::move(promise));
 }
 
 void RootDb::store_block_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
-  if (handle->moved_to_archive()) {
-    td::actor::send_closure(new_archive_db_, &ArchiveManager::write_handle, std::move(handle), std::move(promise));
-  } else {
-    td::actor::send_closure(block_db_, &BlockDb::store_block_handle, std::move(handle), std::move(promise));
-  }
+  td::actor::send_closure(archive_db_, &ArchiveManager::update_handle, std::move(handle), std::move(promise));
 }
 
 void RootDb::get_block_handle(BlockIdExt id, td::Promise<BlockHandle> promise) {
-  auto P = td::PromiseCreator::lambda(
-      [db = block_db_.get(), id, promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
-        if (R.is_error()) {
-          td::actor::send_closure(db, &BlockDb::get_block_handle, id, std::move(promise));
-        } else {
-          promise.set_value(R.move_as_ok());
-        }
-      });
-  td::actor::send_closure(new_archive_db_, &ArchiveManager::read_handle, id, std::move(P));
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_handle, id, std::move(promise));
 }
 
 void RootDb::try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice> promise) {
@@ -379,24 +319,20 @@ void RootDb::try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice
 }
 
 void RootDb::apply_block(BlockHandle handle, td::Promise<td::Unit> promise) {
-  if (handle->id().id.seqno == 0) {
-    promise.set_value(td::Unit());
-  } else {
-    td::actor::send_closure(lt_db_, &LtDb::add_new_block, handle->id(), handle->logical_time(), handle->unix_time(),
-                            std::move(promise));
-  }
+  td::actor::create_actor<BlockArchiver>("archiver", std::move(handle), archive_db_.get(), std::move(promise))
+      .release();
 }
 
-void RootDb::get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockIdExt> promise) {
-  td::actor::send_closure(lt_db_, &LtDb::get_block_by_lt, account, lt, std::move(promise));
+void RootDb::get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockHandle> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_block_by_lt, account, lt, std::move(promise));
 }
 
-void RootDb::get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockIdExt> promise) {
-  td::actor::send_closure(lt_db_, &LtDb::get_block_by_unix_time, account, ts, std::move(promise));
+void RootDb::get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockHandle> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_block_by_unix_time, account, ts, std::move(promise));
 }
 
-void RootDb::get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockIdExt> promise) {
-  td::actor::send_closure(lt_db_, &LtDb::get_block_by_seqno, account, seqno, std::move(promise));
+void RootDb::get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockHandle> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_block_by_seqno, account, seqno, std::move(promise));
 }
 
 void RootDb::update_init_masterchain_block(BlockIdExt block, td::Promise<td::Unit> promise) {
@@ -451,19 +387,13 @@ void RootDb::get_hardforks(td::Promise<std::vector<BlockIdExt>> promise) {
 
 void RootDb::start_up() {
   cell_db_ = td::actor::create_actor<CellDb>("celldb", actor_id(this), root_path_ + "/celldb/");
-  block_db_ = td::actor::create_actor<BlockDb>("blockdb", actor_id(this), root_path_ + "/blockdb/");
-  file_db_ = td::actor::create_actor<FileDb>("filedb", actor_id(this), root_path_ + "/files/", depth_, false);
-  old_archive_db_ =
-      td::actor::create_actor<FileDb>("filedbarchive", actor_id(this), root_path_ + "/archive/", depth_, true);
-  lt_db_ = td::actor::create_actor<LtDb>("ltdb", actor_id(this), root_path_ + "/ltdb/");
   state_db_ = td::actor::create_actor<StateDb>("statedb", actor_id(this), root_path_ + "/state/");
   static_files_db_ = td::actor::create_actor<StaticFilesDb>("staticfilesdb", actor_id(this), root_path_ + "/static/");
-  new_archive_db_ = td::actor::create_actor<ArchiveManager>("archivemanager", root_path_ + "/archive/");
+  archive_db_ = td::actor::create_actor<ArchiveManager>("archive", actor_id(this), root_path_);
 }
 
-void RootDb::archive(BlockIdExt block_id, td::Promise<td::Unit> promise) {
-  td::actor::create_actor<BlockArchiver>("archiveblock", block_id, actor_id(this), file_db_.get(),
-                                         old_archive_db_.get(), new_archive_db_.get(), std::move(promise))
+void RootDb::archive(BlockHandle handle, td::Promise<td::Unit> promise) {
+  td::actor::create_actor<BlockArchiver>("archiveblock", std::move(handle), archive_db_.get(), std::move(promise))
       .release();
 }
 
@@ -475,57 +405,86 @@ void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise<bool> promise) {
   td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_info_gc, block_id, std::move(promise));
 }
 
-void RootDb::allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise<bool> promise) {
-  ref_id.visit(
-      td::overloaded([&](const fileref::Empty &key) { UNREACHABLE(); },
-                     [&](const fileref::Block &key) {
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_data_gc, key.block_id,
-                                               is_archive, std::move(promise));
-                     },
-                     [&](const fileref::ZeroState &key) {
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_zero_state_file_gc,
-                                               key.block_id, std::move(promise));
-                     },
-                     [&](const fileref::PersistentState &key) {
-                       CHECK(is_archive);
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_persistent_state_file_gc,
-                                               key.block_id, key.masterchain_block_id, std::move(promise));
-                     },
-                     [&](const fileref::Proof &key) {
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_proof_gc,
-                                               key.block_id, is_archive, std::move(promise));
-                     },
-                     [&](const fileref::ProofLink &key) {
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_proof_link_gc,
-                                               key.block_id, is_archive, std::move(promise));
-                     },
-                     [&](const fileref::Signatures &key) {
-                       CHECK(!is_archive);
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_signatures_gc,
-                                               key.block_id, std::move(promise));
-                     },
-                     [&](const fileref::Candidate &key) {
-                       CHECK(!is_archive);
-                       td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_candidate_gc,
-                                               key.block_id, std::move(promise));
-                     },
-                     [&](const fileref::BlockInfo &key) { UNREACHABLE(); }));
-}
-
 void RootDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
   auto merger = StatsMerger::create(std::move(promise));
-
-  td::actor::send_closure(file_db_, &FileDb::prepare_stats, merger.make_promise("filedb."));
-  td::actor::send_closure(old_archive_db_, &FileDb::prepare_stats, merger.make_promise("archivedb."));
 }
 
 void RootDb::truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) {
   td::MultiPromise mp;
   auto ig = mp.init_guard();
   ig.add_promise(std::move(promise));
+}
 
-  td::actor::send_closure(lt_db_, &LtDb::truncate, state, ig.get_promise());
-  td::actor::send_closure(block_db_, &BlockDb::truncate, state, ig.get_promise());
+void RootDb::add_key_block_proof(td::Ref<Proof> proof, td::Promise<td::Unit> promise) {
+  auto i = proof->get_basic_header_info().move_as_ok();
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_key_block_proof, i.utime, proof->block_id().seqno(),
+                          i.end_lt, fileref::Proof{proof->block_id()}, proof->data(), std::move(promise));
+}
+
+void RootDb::add_key_block_proof_link(td::Ref<ProofLink> proof, td::Promise<td::Unit> promise) {
+  auto i = proof->get_basic_header_info().move_as_ok();
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_key_block_proof, i.utime, proof->block_id().seqno(),
+                          i.end_lt, fileref::ProofLink{proof->block_id()}, proof->data(), std::move(promise));
+}
+void RootDb::get_key_block_proof(BlockIdExt block_id, td::Promise<td::Ref<Proof>> promise) {
+  auto P = td::PromiseCreator::lambda([block_id, promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_error()) {
+      promise.set_error(R.move_as_error());
+    } else {
+      promise.set_result(create_proof(block_id, R.move_as_ok()));
+    }
+  });
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::Proof{block_id}, std::move(P));
+}
+void RootDb::get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::Ref<ProofLink>> promise) {
+  auto P = td::PromiseCreator::lambda([block_id, promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_error()) {
+      promise.set_error(R.move_as_error());
+    } else {
+      promise.set_result(create_proof_link(block_id, R.move_as_ok()));
+    }
+  });
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::Proof{block_id}, std::move(P));
+}
+
+void RootDb::check_key_block_proof_exists(BlockIdExt block_id, td::Promise<bool> promise) {
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_error()) {
+      promise.set_result(false);
+    } else {
+      promise.set_result(true);
+    }
+  });
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::Proof{block_id}, std::move(P));
+}
+void RootDb::check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise<bool> promise) {
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_error()) {
+      promise.set_result(false);
+    } else {
+      promise.set_result(true);
+    }
+  });
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::ProofLink{block_id},
+                          std::move(P));
+}
+
+void RootDb::get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_id, masterchain_seqno, std::move(promise));
+}
+
+void RootDb::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                               td::Promise<td::BufferSlice> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_slice, archive_id, offset, limit,
+                          std::move(promise));
+}
+
+void RootDb::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise));
+}
+
+void RootDb::run_gc(UnixTime ts) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, ts);
 }
 
 }  // namespace validator

validator/db/rootdb.hpp

@@ -22,13 +22,10 @@
 #include "td/db/KeyValueAsync.h"
 #include "ton/ton-types.h"
 
-#include "blockdb.hpp"
 #include "celldb.hpp"
-#include "filedb.hpp"
-#include "ltdb.hpp"
 #include "statedb.hpp"
 #include "staticfilesdb.hpp"
-#include "archive-db.hpp"
+#include "archive-manager.hpp"
 
 namespace ton {
 
@@ -85,9 +82,9 @@ class RootDb : public Db {
   void try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice> promise) override;
 
   void apply_block(BlockHandle handle, td::Promise<td::Unit> promise) override;
-  void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockIdExt> promise) override;
-  void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockIdExt> promise) override;
-  void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockIdExt> promise) override;
+  void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockHandle> promise) override;
+  void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockHandle> promise) override;
+  void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockHandle> promise) override;
 
   void update_init_masterchain_block(BlockIdExt block, td::Promise<td::Unit> promise) override;
   void get_init_masterchain_block(td::Promise<BlockIdExt> promise) override;
@@ -108,16 +105,30 @@ class RootDb : public Db {
   void update_hardforks(std::vector<BlockIdExt> blocks, td::Promise<td::Unit> promise) override;
   void get_hardforks(td::Promise<std::vector<BlockIdExt>> promise) override;
 
-  void archive(BlockIdExt block_id, td::Promise<td::Unit> promise) override;
+  void archive(BlockHandle handle, td::Promise<td::Unit> promise) override;
 
   void allow_state_gc(BlockIdExt block_id, td::Promise<bool> promise);
   void allow_block_gc(BlockIdExt block_id, td::Promise<bool> promise);
-  void allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise<bool> promise);
+  //void allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise<bool> promise);
 
   void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) override;
 
   void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) override;
 
+  void add_key_block_proof(td::Ref<Proof> proof, td::Promise<td::Unit> promise) override;
+  void add_key_block_proof_link(td::Ref<ProofLink> proof_link, td::Promise<td::Unit> promise) override;
+  void get_key_block_proof(BlockIdExt block_id, td::Promise<td::Ref<Proof>> promise) override;
+  void get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::Ref<ProofLink>> promise) override;
+  void check_key_block_proof_exists(BlockIdExt block_id, td::Promise<bool> promise) override;
+  void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise<bool> promise) override;
+
+  void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) override;
+  void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                         td::Promise<td::BufferSlice> promise) override;
+  void set_async_mode(bool mode, td::Promise<td::Unit> promise) override;
+
+  void run_gc(UnixTime ts) override;
+
  private:
   td::actor::ActorId<ValidatorManager> validator_manager_;
 
@@ -125,13 +136,9 @@ class RootDb : public Db {
   td::uint32 depth_;
 
   td::actor::ActorOwn<CellDb> cell_db_;
-  td::actor::ActorOwn<BlockDb> block_db_;
-  td::actor::ActorOwn<FileDb> file_db_;
-  td::actor::ActorOwn<FileDb> old_archive_db_;
-  td::actor::ActorOwn<LtDb> lt_db_;
   td::actor::ActorOwn<StateDb> state_db_;
   td::actor::ActorOwn<StaticFilesDb> static_files_db_;
-  td::actor::ActorOwn<ArchiveManager> new_archive_db_;
+  td::actor::ActorOwn<ArchiveManager> archive_db_;
 };
 
 }  // namespace validator

validator/db/statedb.cpp

@@ -222,6 +222,22 @@ StateDb::StateDb(td::actor::ActorId<RootDb> root_db, std::string db_path) : root
 
 void StateDb::start_up() {
   kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
+
+  std::string value;
+  auto R = kv_->get(create_serialize_tl_object<ton_api::db_state_key_dbVersion>(), value);
+  R.ensure();
+  if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) {
+    auto F = fetch_tl_object<ton_api::db_state_dbVersion>(value, true);
+    F.ensure();
+    auto f = F.move_as_ok();
+    CHECK(f->version_ == 2);
+  } else {
+    kv_->begin_transaction().ensure();
+    kv_->set(create_serialize_tl_object<ton_api::db_state_key_dbVersion>(),
+             create_serialize_tl_object<ton_api::db_state_dbVersion>(2))
+        .ensure();
+    kv_->commit_transaction().ensure();
+  }
 }
 
 }  // namespace validator

validator/db/statedb.hpp

@@ -50,6 +50,9 @@ class StateDb : public td::actor::Actor {
   void update_hardforks(std::vector<BlockIdExt> blocks, td::Promise<td::Unit> promise);
   void get_hardforks(td::Promise<std::vector<BlockIdExt>> promise);
 
+  void update_db_version(td::uint32 version, td::Promise<td::Unit> promise);
+  void get_db_version(td::Promise<td::uint32> promise);
+
   StateDb(td::actor::ActorId<RootDb> root_db, std::string path);
 
   void start_up() override;

validator/downloaders/download-state.cpp

@@ -194,11 +194,13 @@ void DownloadShardState::written_shard_state(td::Ref<ShardState> state) {
   handle_->set_applied();
   handle_->set_split(state_->before_split());
 
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result<td::Unit> R) {
+    CHECK(handle->handle_moved_to_archive());
+    CHECK(handle->moved_to_archive())
     R.ensure();
     td::actor::send_closure(SelfId, &DownloadShardState::written_block_handle);
   });
-  handle_->flush(manager_, handle_, std::move(P));
+  td::actor::send_closure(manager_, &ValidatorManager::archive, handle_, std::move(P));
 }
 
 void DownloadShardState::written_block_handle() {

validator/fabric.h

@@ -54,8 +54,9 @@ void run_fake_accept_block_query(BlockIdExt id, td::Ref<BlockData> data, std::ve
                                  td::Promise<td::Unit> promise);
 void run_hardfork_accept_block_query(BlockIdExt id, td::Ref<BlockData> data,
                                      td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise);
-void run_apply_block_query(BlockIdExt id, td::Ref<BlockData> block, td::actor::ActorId<ValidatorManager> manager,
-                           td::Timestamp timeout, td::Promise<td::Unit> promise);
+void run_apply_block_query(BlockIdExt id, td::Ref<BlockData> block, BlockIdExt masterchain_block_id,
+                           td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
+                           td::Promise<td::Unit> promise);
 void run_check_proof_query(BlockIdExt id, td::Ref<Proof> proof, td::actor::ActorId<ValidatorManager> manager,
                            td::Timestamp timeout, td::Promise<BlockHandle> promise, bool skip_check_signatures = false);
 void run_check_proof_query(BlockIdExt id, td::Ref<Proof> proof, td::actor::ActorId<ValidatorManager> manager,

validator/full-node-master.cpp

@@ -309,6 +309,26 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo
   promise.set_value(create_serialize_tl_object<ton_api::tonNode_capabilities>(proto_version(), proto_capabilities()));
 }
 
+void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query,
+                                       td::Promise<td::BufferSlice> promise) {
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::uint64> R) mutable {
+        if (R.is_error()) {
+          promise.set_value(create_serialize_tl_object<ton_api::tonNode_archiveNotFound>());
+        } else {
+          promise.set_value(create_serialize_tl_object<ton_api::tonNode_archiveInfo>(R.move_as_ok()));
+        }
+      });
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_,
+                          std::move(P));
+}
+
+void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query,
+                                       td::Promise<td::BufferSlice> promise) {
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_,
+                          query.offset_, query.max_size_, std::move(promise));
+}
+
 void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_slave_sendExtMessage &query,
                                        td::Promise<td::BufferSlice> promise) {
   td::actor::send_closure(

validator/full-node-master.hpp

@@ -72,6 +72,10 @@ class FullNodeMasterImpl : public FullNodeMaster {
                      td::Promise<td::BufferSlice> promise);
   void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_slave_sendExtMessage &query,
                      td::Promise<td::BufferSlice> promise);
+  void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query,
+                     td::Promise<td::BufferSlice> promise);
+  void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query,
+                     td::Promise<td::BufferSlice> promise);
   // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query,
   //                   td::Promise<td::BufferSlice> promise);
   void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise<td::BufferSlice> promise);

validator/full-node-shard.cpp

@@ -31,6 +31,7 @@
 #include "net/download-state.hpp"
 #include "net/download-proof.hpp"
 #include "net/get-next-key-blocks.hpp"
+#include "net/download-archive-slice.hpp"
 
 #include "td/utils/Random.h"
 
@@ -146,6 +147,7 @@ void FullNodeShardImpl::got_next_block(td::Result<BlockHandle> R) {
 }
 
 void FullNodeShardImpl::get_next_block() {
+  //return;
   attempt_++;
   auto P = td::PromiseCreator::lambda([validator_manager = validator_manager_, attempt = attempt_,
                                        block_id = handle_->id(), SelfId = actor_id(this)](td::Result<ReceivedBlock> R) {
@@ -450,6 +452,26 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod
   promise.set_value(create_serialize_tl_object<ton_api::tonNode_capabilities>(proto_version(), proto_capabilities()));
 }
 
+void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query,
+                                      td::Promise<td::BufferSlice> promise) {
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::uint64> R) mutable {
+        if (R.is_error()) {
+          promise.set_value(create_serialize_tl_object<ton_api::tonNode_archiveNotFound>());
+        } else {
+          promise.set_value(create_serialize_tl_object<ton_api::tonNode_archiveInfo>(R.move_as_ok()));
+        }
+      });
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_,
+                          std::move(P));
+}
+
+void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query,
+                                      td::Promise<td::BufferSlice> promise) {
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_,
+                          query.offset_, query.max_size_, std::move(promise));
+}
+
 void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query,
                                       td::Promise<td::BufferSlice> promise) {
   auto B = fetch_tl_object<ton_api::Function>(std::move(query), true);
@@ -637,6 +659,15 @@ void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp t
       .release();
 }
 
+void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                                         td::Promise<std::string> promise) {
+  auto &b = choose_neighbour();
+  td::actor::create_actor<DownloadArchiveSlice>(
+      "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), timeout,
+      validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise)))
+      .release();
+}
+
 void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
   CHECK(!handle_);
   handle_ = std::move(handle);
@@ -741,7 +772,7 @@ void FullNodeShardImpl::update_validators(std::vector<PublicKeyHash> public_key_
     authorized_keys.emplace(key, overlay::Overlays::max_fec_broadcast_size());
   }
 
-  rules_ = overlay::OverlayPrivacyRules{overlay::Overlays::max_simple_broadcast_size(), std::move(authorized_keys)};
+  rules_ = overlay::OverlayPrivacyRules{1 << 14, std::move(authorized_keys)};
   td::actor::send_closure(overlays_, &overlay::Overlays::set_privacy_rules, adnl_id_, overlay_id_, rules_);
 
   if (update_cert) {

validator/full-node-shard.h

@@ -55,6 +55,8 @@ class FullNodeShard : public td::actor::Actor {
                                          td::Promise<td::BufferSlice> promise) = 0;
   virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout,
                                    td::Promise<std::vector<BlockIdExt>> promise) = 0;
+  virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                                td::Promise<std::string> promise) = 0;
 
   virtual void set_handle(BlockHandle handle, td::Promise<td::Unit> promise) = 0;
 

validator/full-node-shard.hpp

@@ -61,7 +61,7 @@ class FullNodeShardImpl : public FullNodeShard {
     return 1;
   }
   static constexpr td::uint32 proto_version() {
-    return 1;
+    return 2;
   }
   static constexpr td::uint64 proto_capabilities() {
     return 0;
@@ -120,6 +120,10 @@ class FullNodeShardImpl : public FullNodeShard {
                      td::Promise<td::BufferSlice> promise);
   void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query,
                      td::Promise<td::BufferSlice> promise);
+  void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query,
+                     td::Promise<td::BufferSlice> promise);
+  void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query,
+                     td::Promise<td::BufferSlice> promise);
   // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query,
   //                   td::Promise<td::BufferSlice> promise);
   void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise<td::BufferSlice> promise);
@@ -148,6 +152,8 @@ class FullNodeShardImpl : public FullNodeShard {
                                  td::Promise<td::BufferSlice> promise) override;
   void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout,
                            td::Promise<std::vector<BlockIdExt>> promise) override;
+  void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                        td::Promise<std::string> promise) override;
 
   void set_handle(BlockHandle handle, td::Promise<td::Unit> promise) override;
 

validator/full-node.cpp

@@ -230,6 +230,14 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou
   td::actor::send_closure(shard, &FullNodeShard::get_next_key_blocks, block_id, timeout, std::move(promise));
 }
 
+void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                                    td::Promise<std::string> promise) {
+  auto shard = get_shard(ShardIdFull{masterchainId});
+  CHECK(!shard.empty());
+  td::actor::send_closure(shard, &FullNodeShard::download_archive, masterchain_seqno, std::move(tmp_dir), timeout,
+                          std::move(promise));
+}
+
 td::actor::ActorId<FullNodeShard> FullNodeImpl::get_shard(ShardIdFull shard) {
   while (shards_.count(shard) == 0) {
     if (shard.shard == shardIdAll) {
@@ -392,6 +400,11 @@ void FullNodeImpl::start_up() {
                              td::Promise<std::vector<BlockIdExt>> promise) override {
       td::actor::send_closure(id_, &FullNodeImpl::get_next_key_blocks, block_id, timeout, std::move(promise));
     }
+    void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                          td::Promise<std::string> promise) override {
+      td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout,
+                              std::move(promise));
+    }
 
     void new_key_block(BlockHandle handle) override {
       td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle));

validator/full-node.hpp

@@ -64,6 +64,8 @@ class FullNodeImpl : public FullNode {
   void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout,
                                  td::Promise<td::BufferSlice> promise);
   void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise<std::vector<BlockIdExt>> promise);
+  void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout,
+                        td::Promise<std::string> promise);
 
   void got_key_block_proof(td::Ref<ProofLink> proof);
   void got_zero_block_state(td::Ref<ShardState> state);

validator/impl/accept-block.cpp

@@ -41,14 +41,16 @@ using namespace std::literals::string_literals;
 
 AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
                                    td::Ref<ValidatorSet> validator_set, td::Ref<BlockSignatureSet> signatures,
-                                   bool send_broadcast, td::actor::ActorId<ValidatorManager> manager,
-                                   td::Promise<td::Unit> promise)
+                                   td::Ref<BlockSignatureSet> approve_signatures, bool send_broadcast,
+                                   td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise)
     : id_(id)
     , data_(std::move(data))
     , prev_(std::move(prev))
     , validator_set_(std::move(validator_set))
     , signatures_(std::move(signatures))
+    , approve_signatures_(std::move(approve_signatures))
     , is_fake_(false)
+    , is_fork_(false)
     , send_broadcast_(send_broadcast)
     , manager_(manager)
     , promise_(std::move(promise)) {
@@ -66,6 +68,7 @@ AcceptBlockQuery::AcceptBlockQuery(AcceptBlockQuery::IsFake fake, BlockIdExt id,
     , prev_(std::move(prev))
     , validator_set_(std::move(validator_set))
     , is_fake_(true)
+    , is_fork_(false)
     , send_broadcast_(false)
     , manager_(manager)
     , promise_(std::move(promise)) {
@@ -75,6 +78,74 @@ AcceptBlockQuery::AcceptBlockQuery(AcceptBlockQuery::IsFake fake, BlockIdExt id,
   CHECK(prev_.size() > 0);
 }
 
+AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref<BlockData> data,
+                                   td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise)
+    : id_(id)
+    , data_(std::move(data))
+    , is_fake_(true)
+    , is_fork_(true)
+    , send_broadcast_(false)
+    , manager_(manager)
+    , promise_(std::move(promise)) {
+  state_keep_old_hash_.clear();
+  state_old_hash_.clear();
+  state_hash_.clear();
+}
+
+bool AcceptBlockQuery::precheck_header() {
+  VLOG(VALIDATOR_DEBUG) << "precheck_header()";
+  // 0. sanity check
+  CHECK(data_.not_null());
+  block_root_ = data_->root_cell();
+  if (data_->block_id() != id_) {
+    return fatal_error("incorrect block id in block data: "s + data_->block_id().to_str() + " instead of " +
+                       id_.to_str());
+  }
+  // 1. root hash and file hash check
+  RootHash blk_rhash{block_root_->get_hash().bits()};
+  if (blk_rhash != id_.root_hash) {
+    return fatal_error("block root hash mismatch: expected "s + id_.root_hash.to_hex() + ", found " +
+                       blk_rhash.to_hex());
+  }
+  if (is_fake_ || is_fork_) {
+    FileHash blk_fhash;
+    td::sha256(data_->data().as_slice(), blk_fhash.as_slice());
+    if (blk_fhash != id_.file_hash) {
+      return fatal_error("block file hash mismatch: expected "s + id_.file_hash.to_hex() + ", computed " +
+                         blk_fhash.to_hex());
+    }
+  }
+  // 2. check header fields
+  std::vector<ton::BlockIdExt> prev;
+  ton::BlockIdExt mc_blkid;
+  bool after_split;
+  auto res = block::unpack_block_prev_blk_try(block_root_, id_, prev, mc_blkid, after_split);
+  if (res.is_error()) {
+    return fatal_error("invalid block header in AcceptBlock: "s + res.to_string());
+  }
+  if (is_fork_) {
+    prev_ = prev;
+  } else if (prev_ != prev) {
+    return fatal_error("invalid previous block reference(s) in block header");
+  }
+  // 3. unpack header and check vert_seqno fields
+  block::gen::Block::Record blk;
+  block::gen::BlockInfo::Record info;
+  if (!(tlb::unpack_cell(block_root_, blk) && tlb::unpack_cell(blk.info, info))) {
+    return fatal_error("cannot unpack block header");
+  }
+  if (info.vert_seqno_incr && !is_fork_) {
+    return fatal_error("block header has vert_seqno_incr set in an ordinary AcceptBlock");
+  }
+  if (!info.vert_seqno_incr && is_fork_) {
+    return fatal_error("fork block header has no vert_seqno_incr");
+  }
+  if (is_fork_ && !info.key_block) {
+    return fatal_error("fork block is not a key block");
+  }
+  return true;
+}
+
 bool AcceptBlockQuery::create_new_proof() {
   // 0. check block's root hash
   VLOG(VALIDATOR_DEBUG) << "create_new_proof() : start";
@@ -93,7 +164,7 @@ bool AcceptBlockQuery::create_new_proof() {
   block::CurrencyCollection fees;
   ShardIdFull shard;
   if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(blk.info, info) && !info.version &&
-        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && !info.vert_seq_no &&
+        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) &&
         block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) &&
         tlb::unpack_cell(std::move(blk.extra), extra) && block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) &&
         (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) {
@@ -292,6 +363,26 @@ void AcceptBlockQuery::start_up() {
     fatal_error("no real SignatureSet passed to AcceptBlockQuery");
     return;
   }
+  if (!is_fake_ && is_fork_) {
+    fatal_error("a non-fake AcceptBlockQuery for a forced fork block");
+    return;
+  }
+  if (!is_fork_ && !prev_.size()) {
+    fatal_error("no previous blocks passed to AcceptBlockQuery");
+    return;
+  }
+  if (is_fork_ && !is_masterchain()) {
+    fatal_error("cannot accept a non-masterchain fork block");
+    return;
+  }
+  if (is_fork_ && data_.is_null()) {
+    fatal_error("cannot accept a fork block without explicit data");
+    return;
+  }
+  if (data_.not_null() && !precheck_header()) {
+    fatal_error("invalid block header in AcceptBlock");
+    return;
+  }
 
   td::actor::send_closure(
       manager_, &ValidatorManager::get_block_handle, id_, true, [SelfId = actor_id(this)](td::Result<BlockHandle> R) {
@@ -330,8 +421,8 @@ void AcceptBlockQuery::written_block_data() {
   if (is_fake_) {
     signatures_ = Ref<BlockSignatureSetQ>(create_signature_set(std::vector<BlockSignature>{}));
   }
-  td::actor::send_closure(manager_, &ValidatorManager::set_block_signatures, handle_,
-                          signatures_, [SelfId = actor_id(this)](td::Result<td::Unit> R) {
+  td::actor::send_closure(manager_, &ValidatorManager::set_block_signatures, handle_, signatures_,
+                          [SelfId = actor_id(this)](td::Result<td::Unit> R) {
                             check_send_error(SelfId, R) ||
                                 td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::written_block_signatures);
                           });
@@ -365,8 +456,8 @@ void AcceptBlockQuery::written_block_info() {
     td::actor::send_closure(manager_, &ValidatorManager::wait_prev_block_state, handle_, priority(), timeout_,
                             std::move(P));
   } else {
-    td::actor::send_closure(manager_, &ValidatorManager::wait_block_data, handle_, priority(),
-                            timeout_, [SelfId = actor_id(this)](td::Result<td::Ref<BlockData>> R) {
+    td::actor::send_closure(manager_, &ValidatorManager::wait_block_data, handle_, priority(), timeout_,
+                            [SelfId = actor_id(this)](td::Result<td::Ref<BlockData>> R) {
                               check_send_error(SelfId, R) ||
                                   td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_block_data,
                                                                R.move_as_ok());
@@ -382,6 +473,10 @@ void AcceptBlockQuery::got_block_data(td::Ref<BlockData> data) {
     fatal_error("block data does not contain a root cell");
     return;
   }
+  if (!precheck_header()) {
+    fatal_error("invalid block header in AcceptBlock");
+    return;
+  }
   if (handle_->received()) {
     written_block_data();
   } else {
@@ -406,8 +501,8 @@ void AcceptBlockQuery::got_prev_state(td::Ref<ShardState> state) {
 
   handle_->set_split(state_->before_split());
 
-  td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_,
-                          state_, [SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
+  td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, state_,
+                          [SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
                             check_send_error(SelfId, R) ||
                                 td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::written_state, R.move_as_ok());
                           });
@@ -479,8 +574,8 @@ void AcceptBlockQuery::got_last_mc_block(std::pair<td::Ref<MasterchainState>, Bl
   if (last_mc_id_.id.seqno < mc_blkid_.id.seqno) {
     VLOG(VALIDATOR_DEBUG) << "shardchain block refers to newer masterchain block " << mc_blkid_.to_str()
                           << ", trying to obtain it";
-    td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(),
-                                  timeout_, [SelfId = actor_id(this)](td::Result<Ref<ShardState>> R) {
+    td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(), timeout_,
+                                  [SelfId = actor_id(this)](td::Result<Ref<ShardState>> R) {
                                     check_send_error(SelfId, R) ||
                                         td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_mc_state,
                                                                      R.move_as_ok());
@@ -586,7 +681,7 @@ void AcceptBlockQuery::require_proof_link(BlockIdExt id) {
   CHECK(ton::ShardIdFull(id) == ton::ShardIdFull(id_));
   CHECK(id.id.seqno == id_.id.seqno - 1 - proof_links_.size());
   td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_proof_link_short, id, timeout_,
-                                [ SelfId = actor_id(this), id ](td::Result<Ref<ProofLink>> R) {
+                                [SelfId = actor_id(this), id](td::Result<Ref<ProofLink>> R) {
                                   check_send_error(SelfId, R) ||
                                       td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_proof_link, id,
                                                                    R.move_as_ok());
@@ -785,7 +880,7 @@ void AcceptBlockQuery::written_block_info_2() {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
       check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::applied);
     });
-    run_apply_block_query(handle_->id(), data_, manager_, timeout_, std::move(P));
+    run_apply_block_query(handle_->id(), data_, handle_->id(), manager_, timeout_, std::move(P));
   } else {
     applied();
   }

validator/impl/accept-block.hpp

@@ -47,12 +47,16 @@ using td::Ref;
 class AcceptBlockQuery : public td::actor::Actor {
  public:
   struct IsFake {};
+  struct ForceFork {};
   AcceptBlockQuery(BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
-                   td::Ref<ValidatorSet> validator_set, td::Ref<BlockSignatureSet> signatures, bool send_broadcast,
+                   td::Ref<ValidatorSet> validator_set, td::Ref<BlockSignatureSet> signatures,
+                   td::Ref<BlockSignatureSet> approve_signatures, bool send_broadcast,
                    td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise);
   AcceptBlockQuery(IsFake fake, BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
                    td::Ref<ValidatorSet> validator_set, td::actor::ActorId<ValidatorManager> manager,
                    td::Promise<td::Unit> promise);
+  AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref<BlockData> data,
+                   td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise);
 
  private:
   static constexpr td::uint32 priority() {
@@ -90,7 +94,9 @@ class AcceptBlockQuery : public td::actor::Actor {
   std::vector<BlockIdExt> prev_;
   Ref<ValidatorSetQ> validator_set_;
   Ref<BlockSignatureSetQ> signatures_;
+  Ref<BlockSignatureSetQ> approve_signatures_;
   bool is_fake_;
+  bool is_fork_;
   bool send_broadcast_;
   bool ancestors_split_{false}, is_key_block_{false};
   td::Timestamp timeout_ = td::Timestamp::in(600.0);
@@ -128,6 +134,7 @@ class AcceptBlockQuery : public td::actor::Actor {
   static bool check_send_error(td::actor::ActorId<AcceptBlockQuery> SelfId, td::Result<T>& res) {
     return res.is_error() && check_send_error(std::move(SelfId), res.move_as_error());
   }
+  bool precheck_header();
   bool create_new_proof();
   bool unpack_proof_link(BlockIdExt id, Ref<ProofLink> proof);
 

validator/impl/check-proof.cpp

@@ -165,7 +165,7 @@ bool CheckProof::init_parse(bool is_aux) {
   block::gen::ExtBlkRef::Record mcref;  // _ ExtBlkRef = BlkMasterInfo;
   ShardIdFull shard;
   if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(blk.info, info) && !info.version &&
-        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && !info.vert_seq_no &&
+        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) &&
         block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) &&
         block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) &&
         (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) {

validator/impl/fabric.cpp

@@ -119,7 +119,8 @@ void run_accept_block_query(BlockIdExt id, td::Ref<BlockData> data, std::vector<
                             td::Ref<BlockSignatureSet> approve_signatures, bool send_broadcast,
                             td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise) {
   td::actor::create_actor<AcceptBlockQuery>("accept", id, std::move(data), prev, std::move(validator_set),
-                                            std::move(signatures), send_broadcast, manager, std::move(promise))
+                                            std::move(signatures), std::move(approve_signatures), send_broadcast,
+                                            manager, std::move(promise))
       .release();
 }
 
@@ -134,13 +135,16 @@ void run_fake_accept_block_query(BlockIdExt id, td::Ref<BlockData> data, std::ve
 
 void run_hardfork_accept_block_query(BlockIdExt id, td::Ref<BlockData> data,
                                      td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise) {
-  promise.set_error(td::Status::Error(ErrorCode::error, "not implemented"));
+  td::actor::create_actor<AcceptBlockQuery>("fork/accept", AcceptBlockQuery::ForceFork(), id, std::move(data),
+                                            std::move(manager), std::move(promise))
+      .release();
 }
 
-void run_apply_block_query(BlockIdExt id, td::Ref<BlockData> block, td::actor::ActorId<ValidatorManager> manager,
-                           td::Timestamp timeout, td::Promise<td::Unit> promise) {
-  td::actor::create_actor<ApplyBlock>(PSTRING() << "apply " << id, id, std::move(block), manager, timeout,
-                                      std::move(promise))
+void run_apply_block_query(BlockIdExt id, td::Ref<BlockData> block, BlockIdExt masterchain_block_id,
+                           td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
+                           td::Promise<td::Unit> promise) {
+  td::actor::create_actor<ApplyBlock>(PSTRING() << "apply " << id, id, std::move(block), masterchain_block_id, manager,
+                                      timeout, std::move(promise))
       .release();
 }
 

validator/impl/liteserver.cpp

@@ -192,7 +192,7 @@ void LiteQuery::perform_getMasterchainInfo(int mode) {
   }
   td::actor::send_closure_later(
       manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block,
-      [ Self = actor_id(this), mode ](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res) {
+      [Self = actor_id(this), mode](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
         } else {
@@ -230,7 +230,7 @@ void LiteQuery::perform_getBlock(BlockIdExt blkid) {
     return;
   }
   td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
-                                [ Self = actor_id(this), blkid ](td::Result<Ref<ton::validator::BlockData>> res) {
+                                [Self = actor_id(this), blkid](td::Result<Ref<ton::validator::BlockData>> res) {
                                   if (res.is_error()) {
                                     td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                   } else {
@@ -256,7 +256,7 @@ void LiteQuery::perform_getBlockHeader(BlockIdExt blkid, int mode) {
     return;
   }
   td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
-                                [ Self = actor_id(this), blkid, mode ](td::Result<Ref<ton::validator::BlockData>> res) {
+                                [Self = actor_id(this), blkid, mode](td::Result<Ref<ton::validator::BlockData>> res) {
                                   if (res.is_error()) {
                                     td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                   } else {
@@ -371,7 +371,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) {
   }
   if (blkid.id.seqno) {
     td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
-                                  [ Self = actor_id(this), blkid ](td::Result<Ref<ton::validator::ShardState>> res) {
+                                  [Self = actor_id(this), blkid](td::Result<Ref<ton::validator::ShardState>> res) {
                                     if (res.is_error()) {
                                       td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                     } else {
@@ -381,7 +381,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) {
                                   });
   } else {
     td::actor::send_closure_later(manager_, &ValidatorManager::get_zero_state, blkid,
-                                  [ Self = actor_id(this), blkid ](td::Result<td::BufferSlice> res) {
+                                  [Self = actor_id(this), blkid](td::Result<td::BufferSlice> res) {
                                     if (res.is_error()) {
                                       td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                     } else {
@@ -440,7 +440,7 @@ bool LiteQuery::request_mc_block_data(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<Ref<BlockData>> res) {
+      [Self = actor_id(this), blkid](td::Result<Ref<BlockData>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : "));
@@ -466,7 +466,7 @@ bool LiteQuery::request_mc_proof(BlockIdExt blkid, int mode) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_block_proof_from_db_short, blkid,
-      [ Self = actor_id(this), blkid, mode ](td::Result<Ref<Proof>> res) {
+      [Self = actor_id(this), blkid, mode](td::Result<Ref<Proof>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load proof for "s + blkid.to_str() + " : "));
@@ -488,7 +488,7 @@ bool LiteQuery::request_mc_block_state(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<Ref<ShardState>> res) {
+      [Self = actor_id(this), blkid](td::Result<Ref<ShardState>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : "));
@@ -519,7 +519,7 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<Ref<ShardState>> res) {
+      [Self = actor_id(this), blkid](td::Result<Ref<ShardState>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : "));
@@ -541,7 +541,7 @@ bool LiteQuery::request_block_data(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<Ref<BlockData>> res) {
+      [Self = actor_id(this), blkid](td::Result<Ref<BlockData>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : "));
@@ -563,7 +563,7 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_block_proof_link_from_db_short, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<Ref<ProofLink>> res) {
+      [Self = actor_id(this), blkid](td::Result<Ref<ProofLink>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : "));
@@ -588,7 +588,7 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) {
   ++pending_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_zero_state, blkid,
-      [ Self = actor_id(this), blkid ](td::Result<td::BufferSlice> res) {
+      [Self = actor_id(this), blkid](td::Result<td::BufferSlice> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query,
                                   res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : "));
@@ -632,7 +632,7 @@ void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain,
     LOG(INFO) << "sending a get_top_masterchain_state_block query to manager";
     td::actor::send_closure_later(
         manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block,
-        [Self = actor_id(this)](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res)->void {
+        [Self = actor_id(this)](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res) -> void {
           if (res.is_error()) {
             td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
           } else {
@@ -1067,14 +1067,14 @@ void LiteQuery::continue_getTransactions(unsigned remaining, bool exact) {
              << " " << trans_lt_;
   td::actor::send_closure_later(
       manager_, &ValidatorManager::get_block_by_lt_from_db, ton::extract_addr_prefix(acc_workchain_, acc_addr_),
-      trans_lt_, [ Self = actor_id(this), remaining, manager = manager_ ](td::Result<BlockIdExt> res) {
+      trans_lt_, [Self = actor_id(this), remaining, manager = manager_](td::Result<BlockHandle> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, res.move_as_error(), ton::BlockIdExt{});
         } else {
-          auto blkid = res.move_as_ok();
-          LOG(DEBUG) << "requesting data for block " << blkid.to_str();
-          td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db_short, blkid,
-                                        [Self, blkid, remaining](td::Result<Ref<BlockData>> res) {
+          auto handle = res.move_as_ok();
+          LOG(DEBUG) << "requesting data for block " << handle->id().to_str();
+          td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle,
+                                        [Self, blkid = handle->id(), remaining](td::Result<Ref<BlockData>> res) {
                                           if (res.is_error()) {
                                             td::actor::send_closure(Self, &LiteQuery::abort_getTransactions,
                                                                     res.move_as_error(), blkid);
@@ -1141,7 +1141,7 @@ void LiteQuery::perform_getShardInfo(BlockIdExt blkid, ShardIdFull shard, bool e
 void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector<int> param_list) {
   LOG(INFO) << "started a getConfigParams(" << blkid.to_str() << ", " << mode << ", <list of " << param_list.size()
             << " parameters>) liteserver query";
-  set_continuation([ this, mode, param_list = std::move(param_list) ]() mutable {
+  set_continuation([this, mode, param_list = std::move(param_list)]() mutable {
     continue_getConfigParams(mode, std::move(param_list));
   });
   request_mc_block_data_state(blkid);
@@ -1294,14 +1294,14 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni
   LOG(INFO) << "performing a lookupBlock(" << blkid.to_str() << ", " << mode << ", " << lt << ", " << utime
             << ") query";
   auto P = td::PromiseCreator::lambda(
-      [ Self = actor_id(this), manager = manager_, mode = (mode >> 4) ](td::Result<BlockIdExt> res) {
+      [Self = actor_id(this), manager = manager_, mode = (mode >> 4)](td::Result<BlockHandle> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
         } else {
-          auto blkid = res.move_as_ok();
-          LOG(DEBUG) << "requesting data for block " << blkid.to_str();
-          td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db_short, blkid,
-                                        [Self, blkid, mode](td::Result<Ref<BlockData>> res) {
+          auto handle = res.move_as_ok();
+          LOG(DEBUG) << "requesting data for block " << handle->id().to_str();
+          td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle,
+                                        [Self, blkid = handle->id(), mode](td::Result<Ref<BlockData>> res) {
                                           if (res.is_error()) {
                                             td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                           } else {
@@ -1449,7 +1449,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to,
     if (mode & 0x1000) {
       BlockIdExt bblk = (from.seqno() > to.seqno()) ? from : to;
       td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, bblk,
-                                    [ Self = actor_id(this), from, to, bblk, mode ](td::Result<Ref<ShardState>> res) {
+                                    [Self = actor_id(this), from, to, bblk, mode](td::Result<Ref<ShardState>> res) {
                                       if (res.is_error()) {
                                         td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                       } else {
@@ -1461,7 +1461,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to,
     } else {
       td::actor::send_closure_later(
           manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block,
-          [ Self = actor_id(this), from, to, mode ](td::Result<std::pair<Ref<MasterchainState>, BlockIdExt>> res) {
+          [Self = actor_id(this), from, to, mode](td::Result<std::pair<Ref<MasterchainState>, BlockIdExt>> res) {
             if (res.is_error()) {
               td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
             } else {
@@ -1474,7 +1474,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to,
   } else if (mode & 2) {
     td::actor::send_closure_later(
         manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block,
-        [ Self = actor_id(this), from, mode ](td::Result<std::pair<Ref<MasterchainState>, BlockIdExt>> res) {
+        [Self = actor_id(this), from, mode](td::Result<std::pair<Ref<MasterchainState>, BlockIdExt>> res) {
           if (res.is_error()) {
             td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
           } else {
@@ -1485,7 +1485,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to,
         });
   } else {
     td::actor::send_closure_later(manager_, &ton::validator::ValidatorManager::get_shard_client_state, false,
-                                  [ Self = actor_id(this), from, mode ](td::Result<BlockIdExt> res) {
+                                  [Self = actor_id(this), from, mode](td::Result<BlockIdExt> res) {
                                     if (res.is_error()) {
                                       td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
                                     } else {

validator/impl/proof.cpp

@@ -88,6 +88,7 @@ td::Result<ProofLink::BasicHeaderInfo> ProofLinkQ::get_basic_header_info() const
     }
     res.cc_seqno = info.gen_catchain_seqno;
     res.utime = info.gen_utime;
+    res.end_lt = info.end_lt;
     res.validator_set_hash = info.gen_validator_list_hash_short;
     res.prev_key_mc_seqno = info.prev_key_block_seqno;
     return res;

validator/impl/validate-query.cpp

@@ -430,6 +430,7 @@ bool ValidateQuery::init_parse() {
     return reject_query("a non-masterchain block cannot be a key block");
   }
   if (info.vert_seqno_incr) {
+    // what about non-masterchain blocks?
     return reject_query("new blocks cannot have vert_seqno_incr set");
   }
   if (info.after_merge != after_merge_) {
@@ -1620,12 +1621,13 @@ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block:
                                   << " has unchanged catchain seqno " << cc_seqno
                                   << ", but it must have been updated for all shards");
   }
-  if (!cc_updated && !info.before_merge_ && old_before_merge && !workchain_created) {
+  bool bm_cleared = !info.before_merge_ && old_before_merge;
+  if (!cc_updated && bm_cleared && !workchain_created) {
     return reject_query(PSTRING() << "new shard configuration for shard " << shard.to_str()
                                   << " has unchanged catchain seqno " << cc_seqno
                                   << " while the before_merge bit has been cleared");
   }
-  if (cc_updated && (!update_shard_cc_ || (!info.before_merge_ && old_before_merge))) {
+  if (cc_updated && !(update_shard_cc_ || bm_cleared)) {
     return reject_query(PSTRING() << "new shard configuration for shard " << shard.to_str()
                                   << " has increased catchain seqno " << cc_seqno << " without a good reason");
   }

validator/import-db-slice.cpp

@@ -0,0 +1,354 @@
+#include "import-db-slice.hpp"
+#include "validator/db/fileref.hpp"
+#include "td/utils/overloaded.h"
+#include "validator/fabric.h"
+#include "td/actor/MultiPromise.h"
+#include "common/checksum.h"
+#include "td/utils/port/path.h"
+
+namespace ton {
+
+namespace validator {
+
+ArchiveImporter::ArchiveImporter(std::string path, td::Ref<MasterchainState> state, BlockSeqno shard_client_seqno,
+                                 td::Ref<ValidatorManagerOptions> opts, td::actor::ActorId<ValidatorManager> manager,
+                                 td::Promise<std::vector<BlockSeqno>> promise)
+    : path_(std::move(path))
+    , state_(std::move(state))
+    , shard_client_seqno_(shard_client_seqno)
+    , opts_(std::move(opts))
+    , manager_(manager)
+    , promise_(std::move(promise)) {
+}
+
+void ArchiveImporter::start_up() {
+  auto R = Package::open(path_, false, false);
+  if (R.is_error()) {
+    abort_query(R.move_as_error());
+    return;
+  }
+  package_ = std::make_shared<Package>(R.move_as_ok());
+
+  bool fail = false;
+  package_->iterate([&](std::string filename, td::BufferSlice data, td::uint64 offset) -> bool {
+    auto F = FileReference::create(filename);
+    if (F.is_error()) {
+      abort_query(F.move_as_error());
+      fail = true;
+      return false;
+    }
+    auto f = F.move_as_ok();
+
+    BlockIdExt b;
+    bool is_proof = false;
+    bool ignore = true;
+
+    f.ref().visit(td::overloaded(
+        [&](const fileref::Proof &p) {
+          b = p.block_id;
+          ignore = !b.is_masterchain();
+          is_proof = true;
+        },
+        [&](const fileref::ProofLink &p) {
+          b = p.block_id;
+          ignore = b.is_masterchain();
+          is_proof = true;
+        },
+        [&](const fileref::Block &p) {
+          b = p.block_id;
+          ignore = false;
+          is_proof = false;
+        },
+        [&](const auto &p) { ignore = true; }));
+
+    if (!ignore) {
+      blocks_[b][is_proof ? 0 : 1] = offset;
+      if (b.is_masterchain()) {
+        masterchain_blocks_[b.seqno()] = b;
+      }
+    }
+    return true;
+  });
+
+  if (fail) {
+    return;
+  }
+
+  if (masterchain_blocks_.size() == 0) {
+    abort_query(td::Status::Error(ErrorCode::notready, "archive does not contain any masterchain blocks"));
+    return;
+  }
+
+  auto seqno = masterchain_blocks_.begin()->first;
+
+  check_masterchain_block(seqno);
+}
+
+void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) {
+  auto it = masterchain_blocks_.find(seqno);
+  if (it == masterchain_blocks_.end()) {
+    if (seqno == 0) {
+      abort_query(td::Status::Error(ErrorCode::notready, "no new blocks"));
+      return;
+    }
+    checked_all_masterchain_blocks(seqno - 1);
+    return;
+  }
+  if (seqno < state_->get_block_id().seqno()) {
+    if (!state_->check_old_mc_block_id(it->second)) {
+      abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id"));
+      return;
+    }
+    check_masterchain_block(seqno + 1);
+  } else if (seqno == state_->get_block_id().seqno()) {
+    if (state_->get_block_id() != it->second) {
+      abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id"));
+      return;
+    }
+    check_masterchain_block(seqno + 1);
+  } else {
+    if (seqno != state_->get_block_id().seqno() + 1) {
+      abort_query(td::Status::Error(ErrorCode::protoviolation, "hole in masterchain seqno"));
+      return;
+    }
+    auto it2 = blocks_.find(it->second);
+    CHECK(it2 != blocks_.end());
+
+    auto R1 = package_->read(it2->second[0]);
+    if (R1.is_error()) {
+      abort_query(R1.move_as_error());
+      return;
+    }
+
+    auto proofR = create_proof(it->second, std::move(R1.move_as_ok().second));
+    if (proofR.is_error()) {
+      abort_query(proofR.move_as_error());
+      return;
+    }
+
+    auto R2 = package_->read(it2->second[1]);
+    if (R2.is_error()) {
+      abort_query(R2.move_as_error());
+      return;
+    }
+
+    if (sha256_bits256(R2.ok().second.as_slice()) != it->second.file_hash) {
+      abort_query(td::Status::Error(ErrorCode::protoviolation, "bad block file hash"));
+      return;
+    }
+    auto dataR = create_block(it->second, std::move(R2.move_as_ok().second));
+    if (dataR.is_error()) {
+      abort_query(dataR.move_as_error());
+      return;
+    }
+
+    auto proof = proofR.move_as_ok();
+    auto data = dataR.move_as_ok();
+
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = state_->get_block_id(),
+                                         data](td::Result<BlockHandle> R) mutable {
+      if (R.is_error()) {
+        td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error());
+        return;
+      }
+      auto handle = R.move_as_ok();
+      CHECK(!handle->merge_before());
+      if (handle->one_prev(true) != id) {
+        td::actor::send_closure(SelfId, &ArchiveImporter::abort_query,
+                                td::Status::Error(ErrorCode::protoviolation, "prev block mismatch"));
+        return;
+      }
+      td::actor::send_closure(SelfId, &ArchiveImporter::checked_masterchain_proof, std::move(handle), std::move(data));
+    });
+
+    run_check_proof_query(it->second, std::move(proof), manager_, td::Timestamp::in(2.0), std::move(P), state_,
+                          opts_->is_hardfork(it->second));
+  }
+}
+
+void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Ref<BlockData> data) {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Unit> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveImporter::applied_masterchain_block, std::move(handle));
+  });
+  run_apply_block_query(handle->id(), std::move(data), handle->id(), manager_, td::Timestamp::in(10.0), std::move(P));
+}
+
+void ArchiveImporter::applied_masterchain_block(BlockHandle handle) {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
+    R.ensure();
+    td::actor::send_closure(SelfId, &ArchiveImporter::got_new_materchain_state,
+                            td::Ref<MasterchainState>(R.move_as_ok()));
+  });
+  td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P));
+}
+
+void ArchiveImporter::got_new_materchain_state(td::Ref<MasterchainState> state) {
+  state_ = std::move(state);
+  check_masterchain_block(state_->get_block_id().seqno() + 1);
+}
+
+void ArchiveImporter::checked_all_masterchain_blocks(BlockSeqno seqno) {
+  max_shard_client_seqno_ = seqno;
+  check_next_shard_client_seqno(shard_client_seqno_ + 1);
+}
+
+void ArchiveImporter::check_next_shard_client_seqno(BlockSeqno seqno) {
+  if (seqno > max_shard_client_seqno_) {
+    finish_query();
+    return;
+  }
+
+  if (seqno == max_shard_client_seqno_) {
+    got_masterchain_state(state_);
+  } else {
+    BlockIdExt b;
+    bool f = state_->get_old_mc_block_id(seqno, b);
+    CHECK(f);
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
+      R.ensure();
+      td::actor::send_closure(SelfId, &ArchiveImporter::got_masterchain_state,
+                              td::Ref<MasterchainState>{R.move_as_ok()});
+    });
+    td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, b, std::move(P));
+  }
+}
+
+void ArchiveImporter::got_masterchain_state(td::Ref<MasterchainState> state) {
+  auto s = state->get_shards();
+
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), seqno = state->get_block_id().seqno()](td::Result<td::Unit> R) {
+        if (R.is_error()) {
+          td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error());
+        } else {
+          td::actor::send_closure(SelfId, &ArchiveImporter::check_next_shard_client_seqno, seqno + 1);
+        }
+      });
+
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise(std::move(P));
+
+  for (auto &shard : s) {
+    apply_shard_block(shard->top_block_id(), state->get_block_id(), ig.get_promise());
+  }
+}
+
+void ArchiveImporter::apply_shard_block(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                        td::Promise<td::Unit> promise) {
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), masterchain_block_id, promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+        R.ensure();
+        td::actor::send_closure(SelfId, &ArchiveImporter::apply_shard_block_cont1, R.move_as_ok(), masterchain_block_id,
+                                std::move(promise));
+      });
+  td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, true, std::move(P));
+}
+
+void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt masterchain_block_id,
+                                              td::Promise<td::Unit> promise) {
+  if (handle->is_applied()) {
+    promise.set_value(td::Unit());
+    return;
+  }
+
+  auto it = blocks_.find(handle->id());
+  if (it == blocks_.end()) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "no proof for shard block"));
+    return;
+  }
+  TRY_RESULT_PROMISE(promise, data, package_->read(it->second[0]));
+  TRY_RESULT_PROMISE(promise, proof, create_proof_link(handle->id(), std::move(data.second)));
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle, masterchain_block_id,
+                                       promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+    if (R.is_error()) {
+      promise.set_error(R.move_as_error());
+    } else {
+      td::actor::send_closure(SelfId, &ArchiveImporter::apply_shard_block_cont2, std::move(handle),
+                              masterchain_block_id, std::move(promise));
+    }
+  });
+  run_check_proof_link_query(handle->id(), std::move(proof), manager_, td::Timestamp::in(10.0), std::move(P));
+}
+
+void ArchiveImporter::apply_shard_block_cont2(BlockHandle handle, BlockIdExt masterchain_block_id,
+                                              td::Promise<td::Unit> promise) {
+  if (handle->is_applied()) {
+    promise.set_value(td::Unit());
+    return;
+  }
+  CHECK(handle->id().seqno() > 0);
+
+  if (!handle->merge_before() && handle->one_prev(true).shard_full() == handle->id().shard_full()) {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle, masterchain_block_id,
+                                         promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+      if (R.is_error()) {
+        promise.set_error(R.move_as_error());
+      } else {
+        td::actor::send_closure(SelfId, &ArchiveImporter::apply_shard_block_cont3, std::move(handle),
+                                masterchain_block_id, std::move(promise));
+      }
+    });
+    apply_shard_block(handle->one_prev(true), masterchain_block_id, std::move(P));
+  } else {
+    td::MultiPromise mp;
+    auto ig = mp.init_guard();
+    ig.add_promise(std::move(promise));
+    check_shard_block_applied(handle->one_prev(true), ig.get_promise());
+    if (handle->merge_before()) {
+      check_shard_block_applied(handle->one_prev(false), ig.get_promise());
+    }
+  }
+}
+
+void ArchiveImporter::apply_shard_block_cont3(BlockHandle handle, BlockIdExt masterchain_block_id,
+                                              td::Promise<td::Unit> promise) {
+  auto it = blocks_.find(handle->id());
+  CHECK(it != blocks_.end());
+  TRY_RESULT_PROMISE(promise, data, package_->read(it->second[1]));
+  if (sha256_bits256(data.second.as_slice()) != handle->id().file_hash) {
+    promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad block file hash"));
+    return;
+  }
+  TRY_RESULT_PROMISE(promise, block, create_block(handle->id(), std::move(data.second)));
+
+  run_apply_block_query(handle->id(), std::move(block), masterchain_block_id, manager_, td::Timestamp::in(10.0),
+                        std::move(promise));
+}
+
+void ArchiveImporter::check_shard_block_applied(BlockIdExt block_id, td::Promise<td::Unit> promise) {
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), promise = std::move(promise)](td::Result<BlockHandle> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error());
+        } else {
+          auto handle = R.move_as_ok();
+          if (!handle->is_applied()) {
+            promise.set_error(td::Status::Error(ErrorCode::notready, "not applied"));
+          } else {
+            promise.set_value(td::Unit());
+          }
+        }
+      });
+  td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, false, std::move(P));
+}
+
+void ArchiveImporter::abort_query(td::Status error) {
+  if (promise_) {
+    promise_.set_error(std::move(error));
+    td::unlink(path_).ensure();
+  }
+  stop();
+}
+void ArchiveImporter::finish_query() {
+  if (promise_) {
+    promise_.set_value(std::vector<BlockSeqno>(state_->get_block_id().seqno(), max_shard_client_seqno_));
+    td::unlink(path_).ensure();
+  }
+  stop();
+}
+
+}  // namespace validator
+
+}  // namespace ton

validator/import-db-slice.hpp

@@ -0,0 +1,54 @@
+#pragma once
+
+#include "td/actor/actor.h"
+#include "validator/interfaces/validator-manager.h"
+#include "validator/db/package.hpp"
+
+namespace ton {
+
+namespace validator {
+
+class ArchiveImporter : public td::actor::Actor {
+ public:
+  ArchiveImporter(std::string path, td::Ref<MasterchainState> state, BlockSeqno shard_client_seqno,
+                  td::Ref<ValidatorManagerOptions> opts, td::actor::ActorId<ValidatorManager> manager,
+                  td::Promise<std::vector<BlockSeqno>> promise);
+  void start_up() override;
+
+  void abort_query(td::Status error);
+  void finish_query();
+
+  void check_masterchain_block(BlockSeqno seqno);
+  void checked_masterchain_proof(BlockHandle handle, td::Ref<BlockData> data);
+  void applied_masterchain_block(BlockHandle handle);
+  void got_new_materchain_state(td::Ref<MasterchainState> state);
+  void checked_all_masterchain_blocks(BlockSeqno seqno);
+
+  void check_next_shard_client_seqno(BlockSeqno seqno);
+  void got_masterchain_state(td::Ref<MasterchainState> state);
+  void apply_shard_block(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise);
+  void apply_shard_block_cont1(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise);
+  void apply_shard_block_cont2(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise);
+  void apply_shard_block_cont3(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise);
+  void check_shard_block_applied(BlockIdExt block_id, td::Promise<td::Unit> promise);
+
+ private:
+  std::string path_;
+  td::Ref<MasterchainState> state_;
+  BlockSeqno shard_client_seqno_;
+  BlockSeqno max_shard_client_seqno_;
+
+  td::Ref<ValidatorManagerOptions> opts_;
+
+  std::shared_ptr<Package> package_;
+
+  td::actor::ActorId<ValidatorManager> manager_;
+  td::Promise<std::vector<BlockSeqno>> promise_;
+
+  std::map<BlockSeqno, BlockIdExt> masterchain_blocks_;
+  std::map<BlockIdExt, std::array<td::uint64, 2>> blocks_;
+};
+
+}  // namespace validator
+
+}  // namespace ton

validator/interfaces/block-handle.h

@@ -32,8 +32,8 @@ struct BlockHandleInterface {
  public:
   virtual BlockIdExt id() const = 0;
   virtual bool received() const = 0;
-  virtual bool moved_to_storage() const = 0;
   virtual bool moved_to_archive() const = 0;
+  virtual bool handle_moved_to_archive() const = 0;
   virtual bool deleted() const = 0;
   virtual bool inited_next_left() const = 0;
   virtual bool inited_next_right() const = 0;
@@ -49,6 +49,7 @@ struct BlockHandleInterface {
   virtual bool inited_split_after() const = 0;
   virtual bool inited_merge_before() const = 0;
   virtual bool inited_is_key_block() const = 0;
+  virtual bool inited_masterchain_ref_block() const = 0;
   virtual bool split_after() const = 0;
   virtual bool merge_before() const = 0;
   virtual bool is_key_block() const = 0;
@@ -60,6 +61,7 @@ struct BlockHandleInterface {
   virtual bool is_zero() const = 0;
   virtual bool is_archived() const = 0;
   virtual bool is_applied() const = 0;
+  virtual BlockSeqno masterchain_ref_block() const = 0;
   virtual std::vector<BlockIdExt> prev() const = 0;
   virtual BlockIdExt one_prev(bool left) const = 0;
   virtual std::vector<BlockIdExt> next() const = 0;
@@ -83,8 +85,8 @@ struct BlockHandleInterface {
   virtual void set_next(BlockIdExt next) = 0;
   virtual void set_prev(BlockIdExt prev) = 0;
   virtual void set_received() = 0;
-  virtual void set_moved_to_storage() = 0;
   virtual void set_moved_to_archive() = 0;
+  virtual void set_handle_moved_to_archive() = 0;
   virtual void set_deleted() = 0;
   virtual void set_split(bool value) = 0;
   virtual void set_merge(bool value) = 0;
@@ -94,6 +96,7 @@ struct BlockHandleInterface {
   virtual void set_deleted_state_boc() = 0;
   virtual void set_archived() = 0;
   virtual void set_applied() = 0;
+  virtual void set_masterchain_ref_block(BlockSeqno seqno) = 0;
 
   virtual void unsafe_clear_applied() = 0;
   virtual void unsafe_clear_next() = 0;

validator/interfaces/db.h

@@ -72,9 +72,9 @@ class Db : public td::actor::Actor {
   virtual void get_block_handle(BlockIdExt id, td::Promise<BlockHandle> promise) = 0;
 
   virtual void apply_block(BlockHandle handle, td::Promise<td::Unit> promise) = 0;
-  virtual void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockIdExt> promise) = 0;
-  virtual void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockIdExt> promise) = 0;
-  virtual void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockIdExt> promise) = 0;
+  virtual void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockHandle> promise) = 0;
+  virtual void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise<BlockHandle> promise) = 0;
+  virtual void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise<BlockHandle> promise) = 0;
 
   virtual void update_init_masterchain_block(BlockIdExt block, td::Promise<td::Unit> promise) = 0;
   virtual void get_init_masterchain_block(td::Promise<BlockIdExt> promise) = 0;
@@ -95,11 +95,25 @@ class Db : public td::actor::Actor {
   virtual void update_hardforks(std::vector<BlockIdExt> blocks, td::Promise<td::Unit> promise) = 0;
   virtual void get_hardforks(td::Promise<std::vector<BlockIdExt>> promise) = 0;
 
-  virtual void archive(BlockIdExt block_id, td::Promise<td::Unit> promise) = 0;
+  virtual void archive(BlockHandle handle, td::Promise<td::Unit> promise) = 0;
 
   virtual void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) = 0;
 
   virtual void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) = 0;
+
+  virtual void add_key_block_proof(td::Ref<Proof> proof, td::Promise<td::Unit> promise) = 0;
+  virtual void add_key_block_proof_link(td::Ref<ProofLink> proof_link, td::Promise<td::Unit> promise) = 0;
+  virtual void get_key_block_proof(BlockIdExt block_id, td::Promise<td::Ref<Proof>> promise) = 0;
+  virtual void get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::Ref<ProofLink>> promise) = 0;
+  virtual void check_key_block_proof_exists(BlockIdExt block_id, td::Promise<bool> promise) = 0;
+  virtual void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise<bool> promise) = 0;
+
+  virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) = 0;
+  virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                                 td::Promise<td::BufferSlice> promise) = 0;
+  virtual void set_async_mode(bool mode, td::Promise<td::Unit> promise) = 0;
+
+  virtual void run_gc(UnixTime ts) = 0;
 };
 
 }  // namespace validator

validator/interfaces/proof.h

@@ -29,6 +29,7 @@ class ProofLink : public td::CntObject {
  public:
   struct BasicHeaderInfo {
     UnixTime utime;
+    LogicalTime end_lt;
     CatchainSeqno cc_seqno;
     td::uint32 validator_set_hash;
     BlockSeqno prev_key_mc_seqno;

validator/interfaces/validator-manager.h

@@ -147,6 +147,8 @@ class ValidatorManager : public ValidatorManagerInterface {
   virtual void allow_block_candidate_gc(BlockIdExt block_id, td::Promise<bool> promise) = 0;
   virtual void allow_block_info_gc(BlockIdExt block_id, td::Promise<bool> promise) = 0;
 
+  virtual void archive(BlockHandle handle, td::Promise<td::Unit> promise) = 0;
+
   virtual void check_is_hardfork(BlockIdExt block_id, td::Promise<bool> promise) = 0;
   virtual void get_vertical_seqno(BlockSeqno seqno, td::Promise<td::uint32> promise) = 0;
 

validator/invariants.hpp

@@ -45,7 +45,7 @@ class ValidatorInvariants {
     CHECK(handle->inited_merge_before());
     CHECK(handle->inited_split_after());
     CHECK(handle->inited_prev());
-    CHECK(handle->inited_signatures());
+    CHECK(handle->inited_signatures() || handle->is_applied());
     CHECK(handle->inited_state_root_hash());
     CHECK(handle->inited_logical_time());
     CHECK(handle->inited_unix_time());

validator/manager-disk.cpp

@@ -223,6 +223,42 @@ void ValidatorManagerImpl::get_block_proof(BlockHandle handle, td::Promise<td::B
   td::actor::send_closure(db_, &Db::get_block_proof, handle, std::move(P));
 }
 
+void ValidatorManagerImpl::get_key_block_proof(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Ref<Proof>> R) mutable {
+    if (R.is_error()) {
+      promise.set_error(R.move_as_error());
+    } else {
+      auto B = R.move_as_ok();
+      promise.set_value(B->data());
+    }
+  });
+
+  td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P));
+}
+
+void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {
+  auto P = td::PromiseCreator::lambda(
+      [promise = std::move(promise), block_id, db = db_.get()](td::Result<td::Ref<Proof>> R) mutable {
+        if (R.is_error()) {
+          auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Ref<Proof>> R) mutable {
+            if (R.is_error()) {
+              promise.set_error(R.move_as_error());
+            } else {
+              auto B = R.move_as_ok();
+              promise.set_value(B->data());
+            }
+          });
+
+          td::actor::send_closure(db, &Db::get_key_block_proof, block_id, std::move(P));
+        } else {
+          auto B = R.move_as_ok()->export_as_proof_link().move_as_ok();
+          promise.set_value(B->data());
+        }
+      });
+
+  td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P));
+}
+
 void ValidatorManagerImpl::new_external_message(td::BufferSlice data) {
   auto R = create_ext_message(std::move(data));
   if (R.is_ok()) {
@@ -582,17 +618,17 @@ void ValidatorManagerImpl::get_block_proof_link_from_db_short(BlockIdExt block_i
 }
 
 void ValidatorManagerImpl::get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt,
-                                                   td::Promise<BlockIdExt> promise) {
+                                                   td::Promise<BlockHandle> promise) {
   td::actor::send_closure(db_, &Db::get_block_by_lt, account, lt, std::move(promise));
 }
 
 void ValidatorManagerImpl::get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts,
-                                                          td::Promise<BlockIdExt> promise) {
+                                                          td::Promise<BlockHandle> promise) {
   td::actor::send_closure(db_, &Db::get_block_by_unix_time, account, ts, std::move(promise));
 }
 
 void ValidatorManagerImpl::get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno,
-                                                      td::Promise<BlockIdExt> promise) {
+                                                      td::Promise<BlockHandle> promise) {
   td::actor::send_closure(db_, &Db::get_block_by_seqno, account, seqno, std::move(promise));
 }
 

validator/manager-disk.hpp

@@ -119,6 +119,8 @@ class ValidatorManagerImpl : public ValidatorManager {
   void get_block_proof_link(BlockHandle block_id, td::Promise<td::BufferSlice> promise) override {
     UNREACHABLE();
   }
+  void get_key_block_proof(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) override;
+  void get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) override;
   //void get_block_description(BlockIdExt block_id, td::Promise<BlockDescription> promise) override;
 
   void new_external_message(td::BufferSlice data) override;
@@ -200,11 +202,11 @@ class ValidatorManagerImpl : public ValidatorManager {
   void get_block_proof_link_from_db(BlockHandle handle, td::Promise<td::Ref<ProofLink>> promise) override;
   void get_block_proof_link_from_db_short(BlockIdExt id, td::Promise<td::Ref<ProofLink>> promise) override;
 
-  void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockIdExt> promise) override;
+  void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, td::Promise<BlockHandle> promise) override;
   void get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts,
-                                      td::Promise<BlockIdExt> promise) override;
+                                      td::Promise<BlockHandle> promise) override;
   void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno,
-                                  td::Promise<BlockIdExt> promise) override;
+                                  td::Promise<BlockHandle> promise) override;
 
   // get block handle declared in parent class
   void write_handle(BlockHandle handle, td::Promise<td::Unit> promise) override;
@@ -259,6 +261,14 @@ class ValidatorManagerImpl : public ValidatorManager {
     promise.set_error(td::Status::Error(ErrorCode::error, "download disabled"));
   }
 
+  void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) override {
+    UNREACHABLE();
+  }
+  void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
+                         td::Promise<td::BufferSlice> promise) override {
+    UNREACHABLE();
+  }
+
   void add_shard_block_description(td::Ref<ShardTopBlockDescription> desc);
 
   void register_block_handle(BlockHandle handle, td::Promise<BlockHandle> promise);
@@ -327,6 +337,9 @@ class ValidatorManagerImpl : public ValidatorManager {
   void allow_block_info_gc(BlockIdExt block_id, td::Promise<bool> promise) override {
     promise.set_result(false);
   }
+  void archive(BlockHandle handle, td::Promise<td::Unit> promise) override {
+    td::actor::send_closure(db_, &Db::archive, std::move(handle), std::move(promise));
+  }
   void update_last_known_key_block(BlockHandle handle, bool send_request) override {
   }
   void update_shard_client_block_handle(BlockHandle handle, td::Promise<td::Unit> promise) override {

validator/manager-init.cpp

@@ -104,16 +104,22 @@ void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice
     return;
   }
 
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
+  auto proof_link = pp.move_as_ok();
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof_link](td::Result<BlockHandle> R) {
     if (R.is_error()) {
       LOG(WARNING) << "downloaded proof link failed: " << R.move_as_error();
       td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link);
     } else {
-      td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::try_download_key_blocks, false);
+      auto P = td::PromiseCreator::lambda([SelfId, handle = R.move_as_ok()](td::Result<td::Unit> R) {
+        R.ensure();
+        td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::try_download_key_blocks, false);
+      });
+      td::actor::send_closure(db, &Db::add_key_block_proof_link, proof_link, std::move(P));
     }
   });
 
-  run_check_proof_link_query(handle_->id(), pp.move_as_ok(), manager_, td::Timestamp::in(60.0), std::move(P));
+  run_check_proof_link_query(handle_->id(), proof_link, manager_, td::Timestamp::in(60.0), std::move(P));
 }
 
 void ValidatorManagerMasterchainReiniter::downloaded_zero_state() {
@@ -259,6 +265,8 @@ void ValidatorManagerMasterchainReiniter::download_masterchain_state() {
 
 void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Ref<ShardState> state) {
   state_ = td::Ref<MasterchainState>{std::move(state)};
+  CHECK(handle_->received_state());
+  CHECK(handle_->is_applied());
 
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
     R.ensure();
@@ -452,9 +460,9 @@ void ValidatorManagerMasterchainStarter::got_hardforks(std::vector<BlockIdExt> v
     return;
   }
 
-  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockIdExt> R) {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
     R.ensure();
-    td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::got_truncate_block_id, R.move_as_ok());
+    td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::got_truncate_block_handle, R.move_as_ok());
   });
   td::actor::send_closure(db_, &Db::get_block_by_seqno, AccountIdPrefixFull{masterchainId, 0}, b.seqno() - 1,
                           std::move(P));