liteclient signature check support

1. update liteclient/liteserver. Now liteserver sends signatures of
blocks and liteclient checks them. I.e. liteclient completely checks
received data.
2. validator-engine: more GC options
3. blockchain-explorer: show all block transactions (instead of 256)
4. some bugfixes

adnl/adnl-ext-client.hpp

@@ -74,7 +74,7 @@ class AdnlExtClientImpl : public AdnlExtClient {
   }
 
   void start_up() override {
-    alarm_timestamp() = next_create_at_;
+    alarm();
   }
   void conn_stopped(td::actor::ActorId<AdnlExtConnection> conn) {
     if (!conn_.empty() && conn_.get() == conn) {

blockchain-explorer/blockchain-explorer-query.cpp

@@ -395,8 +395,26 @@ void HttpQueryBlockInfo::got_transactions(td::BufferSlice data) {
                                static_cast<ton::LogicalTime>(T->lt_), T->hash_);
   }
 
-  if (!--pending_queries_) {
+  if (f->incomplete_ && transactions_.size() > 0) {
-    finish_query();
+    const auto &T = *transactions_.rbegin();
+    auto query_3 = ton::serialize_tl_object(
+        ton::create_tl_object<ton::lite_api::liteServer_listBlockTransactions>(
+            ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024,
+            ton::create_tl_object<ton::lite_api::liteServer_transactionId3>(T.addr.addr, T.lt), false, false),
+        true);
+    auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+      if (R.is_error()) {
+        td::actor::send_closure(SelfId, &HttpQueryBlockInfo::abort_query, R.move_as_error_prefix("litequery failed: "));
+      } else {
+        td::actor::send_closure(SelfId, &HttpQueryBlockInfo::got_transactions, R.move_as_ok());
+      }
+    });
+    td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query,
+                            std::move(query_3), std::move(P_3));
+  } else {
+    if (!--pending_queries_) {
+      finish_query();
+    }
   }
 }
 
@@ -597,8 +615,27 @@ void HttpQueryBlockSearch::got_transactions(td::BufferSlice data) {
                                static_cast<ton::LogicalTime>(T->lt_), T->hash_);
   }
 
-  if (!--pending_queries_) {
+  if (f->incomplete_ && transactions_.size() > 0) {
-    finish_query();
+    const auto &T = *transactions_.rbegin();
+    auto query_3 = ton::serialize_tl_object(
+        ton::create_tl_object<ton::lite_api::liteServer_listBlockTransactions>(
+            ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024,
+            ton::create_tl_object<ton::lite_api::liteServer_transactionId3>(T.addr.addr, T.lt), false, false),
+        true);
+    auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+      if (R.is_error()) {
+        td::actor::send_closure(SelfId, &HttpQueryBlockSearch::abort_query,
+                                R.move_as_error_prefix("litequery failed: "));
+      } else {
+        td::actor::send_closure(SelfId, &HttpQueryBlockSearch::got_transactions, R.move_as_ok());
+      }
+    });
+    td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query,
+                            std::move(query_3), std::move(P_3));
+  } else {
+    if (!--pending_queries_) {
+      finish_query();
+    }
   }
 }
 

common/errorcode.h

@@ -20,6 +20,14 @@
 
 namespace ton {
 
-enum ErrorCode : int { failure = 601, error = 602, warning = 603, protoviolation = 621, notready = 651, timeout = 652 };
+enum ErrorCode : int {
+  failure = 601,
+  error = 602,
+  warning = 603,
+  protoviolation = 621,
+  notready = 651,
+  timeout = 652,
+  cancelled = 653
+};
 
 }

crypto/block/block.cpp

@@ -1643,6 +1643,37 @@ td::Status unpack_block_prev_blk_ext(Ref<vm::Cell> block_root, const ton::BlockI
   return td::Status::OK();
 }
 
+td::Status check_block_header(Ref<vm::Cell> block_root, const ton::BlockIdExt& id, ton::Bits256* store_shard_hash_to) {
+  block::gen::Block::Record blk;
+  block::gen::BlockInfo::Record info;
+  ton::ShardIdFull shard;
+  if (!(tlb::unpack_cell(block_root, blk) && tlb::unpack_cell(blk.info, info) && !info.version &&
+        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && !info.vert_seq_no)) {
+    return td::Status::Error("cannot unpack block header");
+  }
+  ton::BlockId hdr_id{shard, (unsigned)info.seq_no};
+  if (id.id != hdr_id) {
+    return td::Status::Error("block header contains block id "s + hdr_id.to_str() + ", expected " + id.id.to_str());
+  }
+  if (id.root_hash != block_root->get_hash().bits()) {
+    return td::Status::Error("block header has incorrect root hash "s + block_root->get_hash().bits().to_hex(256) +
+                             " instead of expected " + id.root_hash.to_hex());
+  }
+  if (info.not_master != !shard.is_masterchain()) {
+    return td::Status::Error("block has invalid not_master flag in its (Merkelized) header");
+  }
+  if (store_shard_hash_to) {
+    vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update};
+    if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4  // merkle update
+          && upd_cs.size_ext() == 0x20228)) {
+      return td::Status::Error("invalid Merkle update in block header");
+    }
+    auto upd_hash = upd_cs.prefetch_ref(1)->get_hash(0);
+    *store_shard_hash_to = upd_hash.bits();
+  }
+  return td::Status::OK();
+}
+
 std::unique_ptr<vm::AugmentedDictionary> get_prev_blocks_dict(Ref<vm::Cell> state_root) {
   block::gen::ShardStateUnsplit::Record info;
   block::gen::McStateExtra::Record extra_info;

crypto/block/block.h

@@ -457,27 +457,47 @@ struct ValueFlow {
 
 std::ostream& operator<<(std::ostream& os, const ValueFlow& vflow);
 
-struct BlkProofLink {
+struct BlockProofLink {
   ton::BlockIdExt from, to;
   bool is_key{false}, is_fwd{false};
-  Ref<vm::Cell> dest_proof, shard_proof, proof;
+  Ref<vm::Cell> dest_proof, state_proof, proof;
   ton::CatchainSeqno cc_seqno{0};
   td::uint32 validator_set_hash{0};
   std::vector<ton::BlockSignature> signatures;
-  BlkProofLink(ton::BlockIdExt _from, ton::BlockIdExt _to, bool _iskey = false)
+  BlockProofLink(ton::BlockIdExt _from, ton::BlockIdExt _to, bool _iskey = false)
       : from(_from), to(_to), is_key(_iskey), is_fwd(to.seqno() > from.seqno()) {
   }
+  bool incomplete() const {
+    return dest_proof.is_null();
+  }
+  td::Status validate() const;
 };
 
-struct BlkProofChain {
+struct BlockProofChain {
   ton::BlockIdExt from, to;
   int mode;
-  std::vector<BlkProofLink> links;
+  bool complete{false}, has_key_block{false}, valid{false};
+  ton::BlockIdExt key_blkid;
+  std::vector<BlockProofLink> links;
   std::size_t link_count() const {
     return links.size();
   }
-  BlkProofChain(ton::BlockIdExt _from, ton::BlockIdExt _to, int _mode) : from(_from), to(_to), mode(_mode) {
+  BlockProofChain(ton::BlockIdExt _from, ton::BlockIdExt _to, int _mode = 0) : from(_from), to(_to), mode(_mode) {
   }
+  BlockProofLink& new_link(const ton::BlockIdExt& cur, const ton::BlockIdExt& next, bool iskey = false) {
+    links.emplace_back(cur, next, iskey);
+    return links.back();
+  }
+  const BlockProofLink& last_link() const {
+    return links.back();
+  }
+  BlockProofLink& last_link() {
+    return links.back();
+  }
+  bool last_link_incomplete() const {
+    return !links.empty() && last_link().incomplete();
+  }
+  td::Status validate();
 };
 
 int filter_out_msg_queue(vm::AugmentedDictionary& out_queue, ton::ShardIdFull old_shard, ton::ShardIdFull subshard);
@@ -529,6 +549,8 @@ td::Status unpack_block_prev_blk_ext(Ref<vm::Cell> block_root, const ton::BlockI
 td::Status unpack_block_prev_blk_try(Ref<vm::Cell> block_root, const ton::BlockIdExt& id,
                                      std::vector<ton::BlockIdExt>& prev, ton::BlockIdExt& mc_blkid, bool& after_split,
                                      ton::BlockIdExt* fetch_blkid = nullptr);
+td::Status check_block_header(Ref<vm::Cell> block_root, const ton::BlockIdExt& id,
+                              ton::Bits256* store_shard_hash_to = nullptr);
 
 std::unique_ptr<vm::AugmentedDictionary> get_prev_blocks_dict(Ref<vm::Cell> state_root);
 bool get_old_mc_block_id(vm::AugmentedDictionary* prev_blocks_dict, ton::BlockSeqno seqno, ton::BlockIdExt& blkid,

crypto/block/check-proof.cpp

@@ -25,8 +25,12 @@
 #include "ton/ton-shard.h"
 
 #include "vm/cells/MerkleProof.h"
+#include "openssl/digest.h"
+#include "Ed25519.h"
 
 namespace block {
+using namespace std::literals::string_literals;
+
 td::Status check_block_header_proof(td::Ref<vm::Cell> root, ton::BlockIdExt blkid, ton::Bits256* store_shard_hash_to,
                                     bool check_state_hash) {
   ton::RootHash vhash{root->get_hash().bits()};
@@ -291,4 +295,234 @@ td::Result<TransactionList::Info> TransactionList::validate() const {
   }
   return std::move(res);
 }
+
+td::Status BlockProofLink::validate() const {
+  if (!(from.is_masterchain_ext() && to.is_masterchain_ext())) {
+    return td::Status::Error("BlockProofLink must have both source and destination blocks in the masterchain");
+  }
+  if (from.seqno() == to.seqno()) {
+    return td::Status::Error("BlockProofLink connects two masterchain blocks "s + from.to_str() + " and " +
+                             to.to_str() + " of equal height");
+  }
+  if (is_fwd != (from.seqno() < to.seqno())) {
+    return td::Status::Error("BlockProofLink from "s + from.to_str() + " to " + to.to_str() +
+                             " is incorrectly declared as a " + (is_fwd ? "forward" : "backward") + " link");
+  }
+  if (dest_proof.is_null() && to.seqno()) {
+    return td::Status::Error("BlockProofLink contains no proof for destination block "s + to.to_str());
+  }
+  if (proof.is_null()) {
+    return td::Status::Error("BlockProofLink contains no proof for source block "s + from.to_str());
+  }
+  if (!is_fwd && state_proof.is_null()) {
+    return td::Status::Error("a backward BlockProofLink contains no proof for the source state of "s + from.to_str());
+  }
+  if (is_fwd && signatures.empty()) {
+    return td::Status::Error("a forward BlockProofLink from "s + from.to_str() + " to " + to.to_str() +
+                             " contains no signatures");
+  }
+  try {
+    // virtualize Merkle proof roots
+    auto vs_root = vm::MerkleProof::virtualize(proof, 1);
+    if (vs_root.is_null()) {
+      return td::Status::Error("BlockProofLink contains an invalid Merkle proof for source block "s + from.to_str());
+    }
+    ton::Bits256 state_hash;
+    if (from.seqno()) {
+      TRY_STATUS(check_block_header(vs_root, from, is_fwd ? nullptr : &state_hash));
+    }
+    auto vd_root = dest_proof.not_null() ? vm::MerkleProof::virtualize(dest_proof, 1) : Ref<vm::Cell>{};
+    if (vd_root.is_null() && to.seqno()) {
+      return td::Status::Error("BlockProofLink contains an invalid Merkle proof for destination block "s + to.to_str());
+    }
+    block::gen::Block::Record blk;
+    block::gen::BlockInfo::Record info;
+    if (to.seqno()) {
+      TRY_STATUS(check_block_header(vd_root, to));
+      if (!(tlb::unpack_cell(vd_root, blk) && tlb::unpack_cell(blk.info, info))) {
+        return td::Status::Error("cannot unpack header for block "s + from.to_str());
+      }
+      if (info.key_block != is_key) {
+        return td::Status::Error(PSTRING() << "incorrect is_key_block value " << is_key << " for destination block "
+                                           << to.to_str());
+      }
+    } else if (!is_key) {
+      // return td::Status::Error("Zerostate destination block "s + to.to_str() + " does not have is_key_block set");
+    }
+    if (!is_fwd) {
+      // check a backward link
+      auto vstate_root = vm::MerkleProof::virtualize(state_proof, 1);
+      if (vstate_root.is_null()) {
+        return td::Status::Error("backward BlockProofLink contains an invalid Merkle proof for source state "s +
+                                 from.to_str());
+      }
+      if (state_hash != vstate_root->get_hash().bits()) {
+        return td::Status::Error("BlockProofLink contains a state proof for "s + from.to_str() +
+                                 " with incorrect root hash");
+      }
+      TRY_RESULT(config, block::ConfigInfo::extract_config(vstate_root, block::ConfigInfo::needPrevBlocks));
+      if (!config->check_old_mc_block_id(to, true)) {
+        return td::Status::Error("cannot check that "s + to.to_str() + " is indeed a previous masterchain block of " +
+                                 from.to_str() + " using the presented Merkle proof of masterchain state");
+      }
+      return td::Status::OK();
+    } else {
+      // check a forward link
+      // extract configuration from source key block or zerostate
+      auto cfg_res = from.seqno() ? block::Config::extract_from_key_block(vs_root, block::ConfigInfo::needValidatorSet)
+                                  : block::Config::extract_from_state(vs_root, block::ConfigInfo::needValidatorSet);
+      if (cfg_res.is_error()) {
+        return td::Status::Error("cannot extract configuration from source key block "s + from.to_str() +
+                                 " of a forward BlockProofLink: " + cfg_res.move_as_error().to_string());
+      }
+      auto config = cfg_res.move_as_ok();
+      // compute validator set
+      ton::ShardIdFull shard{ton::masterchainId};
+      auto nodes = config->compute_validator_set(shard, info.gen_utime, info.gen_catchain_seqno);
+      if (nodes.empty()) {
+        return td::Status::Error(PSTRING()
+                                 << "while checking a forward BlockProofLink: cannot compute validator set for block "
+                                 << to.to_str() << " with utime " << info.gen_utime << " and cc_seqno "
+                                 << info.gen_catchain_seqno << " starting from previous key block " << from.to_str());
+      }
+      // check computed validator set hash
+      auto vset_hash = compute_validator_set_hash(cc_seqno, shard, nodes);
+      if (vset_hash != info.gen_validator_list_hash_short) {
+        return td::Status::Error(
+            PSTRING() << "while checking a forward BlockProofLink: computed validator set for block " << to.to_str()
+                      << " with utime " << info.gen_utime << " and cc_seqno " << info.gen_catchain_seqno
+                      << " starting from previous key block " << from.to_str() << " has hash " << vset_hash
+                      << " different from " << info.gen_validator_list_hash_short << " stated in block header");
+      }
+      // check signatures
+      auto err = check_block_signatures(nodes, signatures, to);
+      if (err.is_error()) {
+        return td::Status::Error("error checking signatures for block "s + to.to_str() +
+                                 " in a forward BlockProofLink: " + err.to_string());
+      }
+      return td::Status::OK();
+    }
+  } catch (vm::VmError& err) {
+    return td::Status::Error("vm error while checking BlockProofLink from "s + from.to_str() + " to " + to.to_str() +
+                             " : " + err.get_msg());
+  } catch (vm::VmVirtError& err) {
+    return td::Status::Error("virtualization error while checking BlockProofLink from "s + from.to_str() + " to " +
+                             to.to_str() + " : " + err.get_msg());
+  }
+}
+
+td::Status BlockProofChain::validate() {
+  valid = false;
+  has_key_block = false;
+  key_blkid.invalidate();
+  if (!(from.is_masterchain_ext() && to.is_masterchain_ext())) {
+    return td::Status::Error("BlockProofChain must have both source and destination blocks in the masterchain");
+  }
+  if (!link_count()) {
+    if (from != to) {
+      return td::Status::Error("BlockProofChain has no links, but its source block "s + from.to_str() +
+                               " and destination block " + to.to_str() + " differ");
+    }
+    valid = true;
+    return td::Status::OK();
+  }
+  ton::BlockIdExt cur = from;
+  int i = 0;
+  for (const auto& link : links) {
+    ++i;
+    if (link.from != cur) {
+      return td::Status::Error(PSTRING() << "link #" << i << " in a BlockProofChain begins with block "
+                                         << link.from.to_str() << " but the previous link ends at different block "
+                                         << cur.to_str());
+    }
+    auto err = link.validate();
+    if (err.is_error()) {
+      return td::Status::Error(PSTRING() << "link #" << i << " in BlockProofChain is invalid: " << err.to_string());
+    }
+    if (link.is_key && (!has_key_block || key_blkid.seqno() < link.to.seqno())) {
+      key_blkid = link.to;
+      has_key_block = true;
+    }
+    cur = link.to;
+  }
+  if (cur != to) {
+    return td::Status::Error("last link of BlockProofChain ends at block "s + cur.to_str() +
+                             " different from declared chain destination block " + to.to_str());
+  }
+  valid = true;
+  return td::Status::OK();
+}
+
+td::Bits256 compute_node_id_short(td::Bits256 ed25519_pubkey) {
+  // pub.ed25519#4813b4c6 key:int256 = PublicKey;
+  struct pubkey {
+    int magic = 0x4813b4c6;
+    unsigned char ed25519_key[32];
+  } PK;
+  std::memcpy(PK.ed25519_key, ed25519_pubkey.data(), 32);
+  static_assert(sizeof(pubkey) == 36, "PublicKey structure is not 36 bytes long");
+  td::Bits256 hash;
+  digest::hash_str<digest::SHA256>(hash.data(), (void*)&PK, sizeof(pubkey));
+  return hash;
+}
+
+td::Status check_block_signatures(const std::vector<ton::ValidatorDescr>& nodes,
+                                  const std::vector<ton::BlockSignature>& signatures, const ton::BlockIdExt& blkid) {
+  if (nodes.empty()) {
+    return td::Status::Error("empty validator public keys set");
+  }
+  if (signatures.empty()) {
+    return td::Status::Error("empty validator signature set");
+  }
+  // compute the string to be signed and its hash
+  unsigned char to_sign[68];
+  td::as<td::uint32>(to_sign) = 0xc50b6e70;  // ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId;
+  memcpy(to_sign + 4, blkid.root_hash.data(), 32);
+  memcpy(to_sign + 36, blkid.file_hash.data(), 32);
+  // unsigned char hash[32];
+  // digest::hash_str<digest::SHA256>(hash, (void*)to_sign, sizeof(to_sign));
+
+  ton::ValidatorWeight total_weight = 0, signed_weight = 0;
+  std::vector<std::pair<td::Bits256, unsigned>> node_map;
+  for (unsigned i = 0; i < nodes.size(); i++) {
+    total_weight += nodes[i].weight;
+    node_map.emplace_back(compute_node_id_short(nodes[i].key), i);
+  }
+  std::sort(node_map.begin(), node_map.end());
+  std::vector<unsigned> seen;
+  for (auto& sig : signatures) {
+    // lookup node in validator set
+    auto& id = sig.node;
+    auto it = std::lower_bound(node_map.begin(), node_map.end(), id,
+                               [](const auto& p, const auto& x) { return p.first < x; });
+    if (it == node_map.end() || it->first != id) {
+      return td::Status::Error("signature set contains unknown NodeIdShort "s + id.to_hex());
+    }
+    unsigned i = it->second;
+    seen.emplace_back(i);
+    // check one signature
+    td::Ed25519::PublicKey pub_key{td::SecureString{nodes.at(i).key.as_slice()}};
+    auto res = pub_key.verify_signature(td::Slice{to_sign, 68}, sig.signature.as_slice());
+    if (res.is_error()) {
+      return res;
+    }
+    signed_weight += nodes[i].weight;
+    if (signed_weight > total_weight) {
+      break;
+    }
+  }
+  std::sort(seen.begin(), seen.end());
+  for (std::size_t i = 1; i < seen.size(); i++) {
+    if (seen[i] == seen[i - 1]) {
+      return td::Status::Error("signature set contains duplicate signature for NodeIdShort "s +
+                               compute_node_id_short(nodes.at(seen[i]).key).to_hex());
+    }
+  }
+  if (3 * signed_weight <= 2 * total_weight) {
+    return td::Status::Error(PSTRING() << "insufficient total signature weight: only " << signed_weight << " out of "
+                                       << total_weight);
+  }
+  return td::Status::OK();
+}
+
 }  // namespace block

crypto/block/check-proof.h

@@ -33,6 +33,9 @@ td::Status check_account_proof(td::Slice proof, ton::BlockIdExt shard_blk, const
 td::Result<td::Bits256> check_state_proof(ton::BlockIdExt blkid, td::Slice proof);
 td::Result<Ref<vm::Cell>> check_extract_state_proof(ton::BlockIdExt blkid, td::Slice proof, td::Slice data);
 
+td::Status check_block_signatures(const std::vector<ton::ValidatorDescr>& nodes,
+                                  const std::vector<ton::BlockSignature>& signatures, const ton::BlockIdExt& blkid);
+
 struct AccountState {
   ton::BlockIdExt blk;
   ton::BlockIdExt shard_blk;

crypto/block/mc-config.cpp

@@ -1003,47 +1003,47 @@ std::vector<ton::BlockId> ShardConfig::get_shard_hash_ids(
   std::vector<ton::BlockId> res;
   bool mcout = mc_shard_hash_.is_null() || !mc_shard_hash_->seqno();  // include masterchain as a shard if seqno > 0
   bool ok = shard_hashes_dict_->check_for_each(
-      [&res, &mcout, mc_shard_hash_ = mc_shard_hash_, &filter](Ref<vm::CellSlice> cs_ref, td::ConstBitPtr key,
+      [&res, &mcout, mc_shard_hash_ = mc_shard_hash_, &filter ](Ref<vm::CellSlice> cs_ref, td::ConstBitPtr key, int n)
-                                                               int n) -> bool {
+          ->bool {
-        int workchain = (int)key.get_int(n);
+            int workchain = (int)key.get_int(n);
-        if (workchain >= 0 && !mcout) {
+            if (workchain >= 0 && !mcout) {
-          if (filter(ton::ShardIdFull{ton::masterchainId}, true)) {
+              if (filter(ton::ShardIdFull{ton::masterchainId}, true)) {
-            res.emplace_back(mc_shard_hash_->blk_.id);
+                res.emplace_back(mc_shard_hash_->blk_.id);
-          }
+              }
-          mcout = true;
+              mcout = true;
-        }
+            }
-        if (!cs_ref->have_refs()) {
+            if (!cs_ref->have_refs()) {
-          return false;
-        }
-        std::stack<std::pair<Ref<vm::Cell>, unsigned long long>> stack;
-        stack.emplace(cs_ref->prefetch_ref(), ton::shardIdAll);
-        while (!stack.empty()) {
-          vm::CellSlice cs{vm::NoVm{}, std::move(stack.top().first)};
-          unsigned long long shard = stack.top().second;
-          stack.pop();
-          int t = (int)cs.fetch_ulong(1);
-          if (t < 0) {
-            return false;
-          }
-          if (!filter(ton::ShardIdFull{workchain, shard}, !t)) {
-            continue;
-          }
-          if (!t) {
-            if (!(cs.advance(4) && cs.have(32))) {
               return false;
             }
-            res.emplace_back(workchain, shard, (int)cs.prefetch_ulong(32));
+            std::stack<std::pair<Ref<vm::Cell>, unsigned long long>> stack;
-            continue;
+            stack.emplace(cs_ref->prefetch_ref(), ton::shardIdAll);
-          }
+            while (!stack.empty()) {
-          unsigned long long delta = (td::lower_bit64(shard) >> 1);
+              vm::CellSlice cs{vm::NoVm{}, std::move(stack.top().first)};
-          if (!delta || cs.size_ext() != 0x20000) {
+              unsigned long long shard = stack.top().second;
-            return false;
+              stack.pop();
-          }
+              int t = (int)cs.fetch_ulong(1);
-          stack.emplace(cs.prefetch_ref(1), shard + delta);
+              if (t < 0) {
-          stack.emplace(cs.prefetch_ref(0), shard - delta);
+                return false;
-        }
+              }
-        return true;
+              if (!filter(ton::ShardIdFull{workchain, shard}, !t)) {
-      },
+                continue;
+              }
+              if (!t) {
+                if (!(cs.advance(4) && cs.have(32))) {
+                  return false;
+                }
+                res.emplace_back(workchain, shard, (int)cs.prefetch_ulong(32));
+                continue;
+              }
+              unsigned long long delta = (td::lower_bit64(shard) >> 1);
+              if (!delta || cs.size_ext() != 0x20000) {
+                return false;
+              }
+              stack.emplace(cs.prefetch_ref(1), shard + delta);
+              stack.emplace(cs.prefetch_ref(0), shard - delta);
+            }
+            return true;
+          },
       true);
   if (!ok) {
     return {};
@@ -1399,8 +1399,8 @@ td::Result<std::vector<ton::StdSmcAddress>> Config::get_special_smartcontracts(b
     return td::Status::Error(-666, "configuration loaded without fundamental smart contract list");
   }
   std::vector<ton::StdSmcAddress> res;
-  if (!special_smc_dict->check_for_each([&res, &without_config, conf_addr = config_addr.bits()](
+  if (!special_smc_dict->check_for_each([&res, &without_config, conf_addr = config_addr.bits() ](
-                                            Ref<vm::CellSlice> cs_ref, td::ConstBitPtr key, int n) {
+          Ref<vm::CellSlice> cs_ref, td::ConstBitPtr key, int n) {
         if (cs_ref->size_ext() || n != 256) {
           return false;
         }
@@ -1565,7 +1565,7 @@ std::vector<ton::ValidatorDescr> Config::do_compute_validator_set(const block::C
                                                                   ton::ShardIdFull shard,
                                                                   const block::ValidatorSet& vset, ton::UnixTime time,
                                                                   ton::CatchainSeqno cc_seqno) {
-  LOG(DEBUG) << "in Config::do_compute_validator_set() for " << shard.to_str() << " ; cc_seqno=" << cc_seqno;
+  // LOG(DEBUG) << "in Config::do_compute_validator_set() for " << shard.to_str() << " ; cc_seqno=" << cc_seqno;
   std::vector<ton::ValidatorDescr> nodes;
   bool is_mc = shard.is_masterchain();
   unsigned count = std::min<unsigned>(is_mc ? vset.main : ccv_conf.shard_val_num, vset.total);

crypto/fift/words.cpp

@@ -43,6 +43,7 @@
 #include "block/block.h"
 
 #include "td/utils/filesystem.h"
+#include "td/utils/misc.h"
 #include "td/utils/optional.h"
 #include "td/utils/PathView.h"
 #include "td/utils/port/thread.h"
@@ -1037,7 +1038,7 @@ void interpret_tuple_index(vm::Stack& stack) {
   if ((td::uint64)idx >= tuple->size()) {
     throw vm::VmError{vm::Excno::range_chk, "array index out of range"};
   }
-  stack.push((*tuple)[idx]);
+  stack.push((*tuple)[td::narrow_cast<size_t>(idx)]);
 }
 
 void interpret_tuple_set(vm::Stack& stack) {
@@ -1047,7 +1048,7 @@ void interpret_tuple_set(vm::Stack& stack) {
   if ((td::uint64)idx >= tuple->size()) {
     throw vm::VmError{vm::Excno::range_chk, "array index out of range"};
   }
-  tuple.write()[idx] = std::move(val);
+  tuple.write()[td::narrow_cast<size_t>(idx)] = std::move(val);
   stack.push_tuple(std::move(tuple));
 }
 
@@ -1095,7 +1096,7 @@ void interpret_allot(vm::Stack& stack) {
   auto n = stack.pop_long_range(0xffffffff);
   Ref<vm::Tuple> ref{true};
   auto& tuple = ref.unique_write();
-  tuple.reserve(n);
+  tuple.reserve(td::narrow_cast<size_t>(n));
   while (n-- > 0) {
     tuple.emplace_back(Ref<vm::Box>{true});
   }

crypto/smartcont/config-code.fc

@@ -129,7 +129,7 @@
   var cs = begin_parse(get_data());
   var cfg_dict = cs~load_ref();
   int kl = 32;
-  cfg_dict~idict_set_ref(kl, -17, begin_cell().store_uint(now() >> 8, 32).end_cell());
+  ;; cfg_dict~idict_set_ref(kl, -17, begin_cell().store_uint(now() >> 16, 32).end_cell());
   var next_vset = cfg_dict.idict_get_ref(kl, 36);
   ifnot (next_vset.null?()) {
     ;; check whether we have to set next_vset as the current validator set

crypto/test/test-cells.cpp

@@ -519,11 +519,11 @@ TEST(base64, main) {
   REGRESSION_VERIFY(os.str());
 }
 
-void check_bits256_scan(std::ostream& os, td::Bits256 a, td::Bits256 b) {
+void check_bits256_scan(std::ostream& stream, td::Bits256 a, td::Bits256 b) {
   auto c = a ^ b;
   auto bit = c.count_leading_zeroes();
   auto bit2 = a.count_matching(b);
-  // os << a.to_hex() << " and " << b.to_hex() << " match in " << bit << " or " << bit2 << " first bits" << std::endl;
+  // stream << a.to_hex() << " and " << b.to_hex() << " match in " << bit << " or " << bit2 << " first bits" << std::endl;
   // std::cerr << a.to_hex() << " and " << b.to_hex() << " match in " << bit << " or " << bit2 << " first bits (a XOR b = " << c.to_hex() << ")" << std::endl;
   CHECK((int)bit >= 0 && bit <= 256);
   for (td::uint32 i = 0; i < bit; i++) {
@@ -533,7 +533,7 @@ void check_bits256_scan(std::ostream& os, td::Bits256 a, td::Bits256 b) {
   CHECK(bit == bit2);
 }
 
-void check_bits_scan(std::ostream& os, td::ConstBitPtr a, bool value) {
+void check_bits_scan(std::ostream& stream, td::ConstBitPtr a, bool value) {
   auto bit = (unsigned)a.scan(value, 256);
   CHECK((int)bit >= 0 && bit <= 256);
   for (td::uint32 i = 0; i < bit; i++) {

crypto/vm/boc.cpp

@@ -717,7 +717,7 @@ td::Result<td::Slice> BagOfCells::get_cell_slice(int idx, td::Slice data) {
     return td::Status::Error(PSLICE() << "invalid index entry [" << offs << "; " << offs_end << "], "
                                       << td::tag("data.size()", data.size()));
   }
-  return data.substr(offs, offs_end - offs);
+  return data.substr(offs, td::narrow_cast<size_t>(offs_end - offs));
 }
 
 td::Result<td::Ref<vm::DataCell>> BagOfCells::deserialize_cell(int idx, td::Slice cells_slice,

crypto/vm/cells/MerkleProof.cpp

@@ -19,6 +19,7 @@
 #include "vm/cells/MerkleProof.h"
 #include "vm/cells/CellBuilder.h"
 #include "vm/cells/CellSlice.h"
+#include "vm/boc.h"
 
 #include "td/utils/HashMap.h"
 #include "td/utils/HashSet.h"
@@ -260,4 +261,39 @@ Ref<Cell> MerkleProof::combine(Ref<Cell> a, Ref<Cell> b) {
   }
   return res.move_as_ok();
 }
+
+MerkleProofBuilder::MerkleProofBuilder(Ref<Cell> root)
+    : usage_tree(std::make_shared<CellUsageTree>()), orig_root(std::move(root)) {
+  usage_root = UsageCell::create(orig_root, usage_tree->root_ptr());
+}
+
+void MerkleProofBuilder::reset(Ref<Cell> root) {
+  usage_tree = std::make_shared<CellUsageTree>();
+  orig_root = std::move(root);
+  usage_root = UsageCell::create(orig_root, usage_tree->root_ptr());
+}
+
+void MerkleProofBuilder::clear() {
+  usage_tree.reset();
+  orig_root.clear();
+  usage_root.clear();
+}
+
+Ref<Cell> MerkleProofBuilder::extract_proof() const {
+  return MerkleProof::generate(orig_root, usage_tree.get());
+}
+
+bool MerkleProofBuilder::extract_proof_to(Ref<Cell> &proof_root) const {
+  return orig_root.not_null() && (proof_root = extract_proof()).not_null();
+}
+
+td::Result<td::BufferSlice> MerkleProofBuilder::extract_proof_boc() const {
+  Ref<Cell> proof_root = extract_proof();
+  if (proof_root.is_null()) {
+    return td::Status::Error("cannot create Merkle proof");
+  } else {
+    return std_boc_serialize(std::move(proof_root));
+  }
+}
+
 }  // namespace vm

crypto/vm/cells/MerkleProof.h

@@ -18,11 +18,13 @@
 */
 #pragma once
 #include "vm/cells/Cell.h"
+#include "td/utils/buffer.h"
 
 #include <utility>
 #include <functional>
 
 namespace vm {
+
 class MerkleProof {
  public:
   using IsPrunnedFunction = std::function<bool(const Ref<Cell> &)>;
@@ -43,4 +45,21 @@ class MerkleProof {
   static Ref<Cell> generate_raw(Ref<Cell> cell, CellUsageTree *usage_tree);
   static Ref<Cell> virtualize_raw(Ref<Cell> cell, Cell::VirtualizationParameters virt);
 };
+
+class MerkleProofBuilder {
+  std::shared_ptr<CellUsageTree> usage_tree;
+  Ref<vm::Cell> orig_root, usage_root;
+
+ public:
+  MerkleProofBuilder(Ref<Cell> root);
+  void reset(Ref<Cell> root);
+  void clear();
+  Ref<Cell> root() const {
+    return usage_root;
+  }
+  Ref<Cell> extract_proof() const;
+  bool extract_proof_to(Ref<Cell> &proof_root) const;
+  td::Result<td::BufferSlice> extract_proof_boc() const;
+};
+
 }  // namespace vm

dht/dht.hpp

@@ -73,7 +73,7 @@ class DhtMember : public Dht {
     return 3;
   }
   static constexpr td::uint32 max_k() {
-    return 100;
+    return 10;
   }
   static constexpr td::uint32 max_a() {
     return 10;
@@ -84,8 +84,9 @@ class DhtMember : public Dht {
   };
 
   static td::actor::ActorOwn<DhtMember> create(adnl::AdnlNodeIdShort id, std::string db_root,
-                                               td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
+                                               td::actor::ActorId<keyring::Keyring> keyring,
-                                               td::uint32 k = 10, td::uint32 a = 3);
+                                               td::actor::ActorId<adnl::Adnl> adnl, td::uint32 k = 10,
+                                               td::uint32 a = 3);
 
   //virtual void update_addr_list(tl_object_ptr<ton_api::adnl_addressList> addr_list) = 0;
   //virtual void add_node(adnl::AdnlNodeIdShort id) = 0;

lite-client/lite-client.cpp

@@ -30,13 +30,14 @@
 #include "adnl/adnl-ext-client.h"
 #include "tl-utils/lite-utils.hpp"
 #include "auto/tl/ton_api_json.h"
-#include "auto/tl/lite_api.h"
+#include "auto/tl/lite_api.hpp"
 #include "td/utils/OptionsParser.h"
 #include "td/utils/Time.h"
 #include "td/utils/filesystem.h"
 #include "td/utils/format.h"
 #include "td/utils/Random.h"
 #include "td/utils/crypto.h"
+#include "td/utils/overloaded.h"
 #include "td/utils/port/signals.h"
 #include "td/utils/port/stacktrace.h"
 #include "td/utils/port/StdStreams.h"
@@ -63,6 +64,7 @@
 #include <iostream>
 #include <sstream>
 
+using namespace std::literals::string_literals;
 using td::Ref;
 
 int verbosity;
@@ -113,6 +115,14 @@ void TestNode::run() {
     remote_addr_.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure();
     remote_public_key_ = ton::PublicKey{cli->id_};
     td::TerminalIO::out() << "using liteserver " << idx << " with addr " << remote_addr_ << "\n";
+    if (gc.validator_ && gc.validator_->zero_state_) {
+      zstate_id_.workchain = gc.validator_->zero_state_->workchain_;
+      if (zstate_id_.workchain != ton::workchainInvalid) {
+        zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_;
+        zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_;
+        td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n";
+      }
+    }
   }
 
   client_ =
@@ -138,7 +148,7 @@ bool TestNode::envelope_send_query(td::BufferSlice query, td::Promise<td::Buffer
     return false;
   }
   auto P = td::PromiseCreator::lambda(
-      [SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+      [ SelfId = actor_id(this), promise = std::move(promise) ](td::Result<td::BufferSlice> R) mutable {
         if (R.is_error()) {
           auto err = R.move_as_error();
           LOG(ERROR) << "failed query: " << err;
@@ -208,7 +218,7 @@ bool TestNode::complete_blkid(ton::BlockId partial_blkid, ton::BlockIdExt& compl
 
 bool TestNode::get_server_time() {
   auto b = ton::serialize_tl_object(ton::create_tl_object<ton::lite_api::liteServer_getTime>(), true);
-  return envelope_send_query(std::move(b), [&, Self = actor_id(this)](td::Result<td::BufferSlice> res) -> void {
+  return envelope_send_query(std::move(b), [&, Self = actor_id(this) ](td::Result<td::BufferSlice> res)->void {
     if (res.is_error()) {
       LOG(ERROR) << "cannot get server time";
       return;
@@ -227,10 +237,10 @@ bool TestNode::get_server_time() {
 
 bool TestNode::get_server_version() {
   auto b = ton::serialize_tl_object(ton::create_tl_object<ton::lite_api::liteServer_getVersion>(), true);
-  return envelope_send_query(std::move(b), [&, Self = actor_id(this)](td::Result<td::BufferSlice> res) -> void {
+  return envelope_send_query(std::move(b), [&, Self = actor_id(this) ](td::Result<td::BufferSlice> res)->void {
+    server_ok_ = false;
     if (res.is_error()) {
       LOG(ERROR) << "cannot get server version and time (server too old?)";
-      return;
     } else {
       auto F = ton::fetch_tl_object<ton::lite_api::liteServer_version>(res.move_as_ok(), true);
       if (F.is_error()) {
@@ -244,14 +254,19 @@ bool TestNode::get_server_version() {
         LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")";
         LOG(WARNING) << "server version is " << (server_version_ >> 8) << "." << (server_version_ & 0xff)
                      << ", capabilities " << server_capabilities_;
+        server_ok_ = (server_version_ >= min_ls_version) && !(~server_capabilities_ & min_ls_capabilities);
       }
     }
+    if (!server_ok_) {
+      LOG(ERROR) << "server version is too old (at least " << (min_ls_version >> 8) << "." << (min_ls_version & 0xff)
+                 << " with capabilities " << min_ls_capabilities << " required), some queries are unavailable";
+    }
   });
 }
 
 bool TestNode::get_server_mc_block_id() {
   auto b = ton::serialize_tl_object(ton::create_tl_object<ton::lite_api::liteServer_getMasterchainInfo>(), true);
-  return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result<td::BufferSlice> res) -> void {
+  return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result<td::BufferSlice> res)->void {
     if (res.is_error()) {
       LOG(ERROR) << "cannot get masterchain info from server";
       return;
@@ -275,13 +290,13 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt
     zstate_id_ = zstateid;
     LOG(INFO) << "zerostate id set to " << zstate_id_.to_str();
   } else if (zstate_id_ != zstateid) {
-    LOG(ERROR) << "fatal: masterchain zero state id suddenly changed: expected " << zstate_id_.to_str() << ", found "
+    LOG(FATAL) << "fatal: masterchain zero state id suddenly changed: expected " << zstate_id_.to_str() << ", found "
                << zstateid.to_str();
-    stop();
+    _exit(3);
     return;
   }
   register_blkid(blkid);
-  //register_blkid(zstateid);
+  register_blkid(ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, 0, zstateid.root_hash, zstateid.file_hash});
   if (!mc_last_id_.is_valid()) {
     mc_last_id_ = blkid;
     request_block(blkid);
@@ -289,12 +304,13 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt
   } else if (mc_last_id_.id.seqno < blkid.id.seqno) {
     mc_last_id_ = blkid;
   }
+  show_new_blkids();
 }
 
 bool TestNode::request_block(ton::BlockIdExt blkid) {
   auto b = ton::serialize_tl_object(
       ton::create_tl_object<ton::lite_api::liteServer_getBlock>(ton::create_tl_lite_block_id(blkid)), true);
-  return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result<td::BufferSlice> res) -> void {
+  return envelope_send_query(std::move(b), [ Self = actor_id(this), blkid ](td::Result<td::BufferSlice> res)->void {
     if (res.is_error()) {
       LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server";
       return;
@@ -319,7 +335,7 @@ bool TestNode::request_block(ton::BlockIdExt blkid) {
 bool TestNode::request_state(ton::BlockIdExt blkid) {
   auto b = ton::serialize_tl_object(
       ton::create_tl_object<ton::lite_api::liteServer_getState>(ton::create_tl_lite_block_id(blkid)), true);
-  return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result<td::BufferSlice> res) -> void {
+  return envelope_send_query(std::move(b), [ Self = actor_id(this), blkid ](td::Result<td::BufferSlice> res)->void {
     if (res.is_error()) {
       LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server";
       return;
@@ -753,6 +769,9 @@ bool TestNode::show_help(std::string command) {
          "ones\n"
          "listblocktrans[rev] <block-id-ext> <count> [<start-account-id> <start-trans-lt>]\tLists block transactions, "
          "starting immediately after or before the specified one\n"
+         "blkproofchain[step] <from-block-id-ext> [<to-block-id-ext>]\tDownloads and checks proof of validity of the "
+         "second "
+         "indicated block (or the last known masterchain block) starting from given block\n"
          "byseqno <workchain> <shard-prefix> <seqno>\tLooks up a block by workchain, shard and seqno, and shows its "
          "header\n"
          "bylt <workchain> <shard-prefix> <lt>\tLooks up a block by workchain, shard and logical time, and shows its "
@@ -834,6 +853,10 @@ bool TestNode::do_parse_line() {
     return parse_block_id_ext(blkid) && parse_uint32(count) &&
            (seekeoln() || (parse_hash(hash) && parse_lt(lt) && (mode |= 128) && seekeoln())) &&
            get_block_transactions(blkid, mode, count, hash, lt);
+  } else if (word == "blkproofchain" || word == "blkproofchainstep") {
+    ton::BlockIdExt blkid2{};
+    return parse_block_id_ext(blkid) && (seekeoln() || parse_block_id_ext(blkid2)) && seekeoln() &&
+           get_block_proof(blkid, blkid2, blkid2.is_valid() + (word == "blkproofchain") * 0x1000);
   } else if (word == "byseqno") {
     return parse_shard_id(shard) && parse_uint32(seqno) && seekeoln() && lookup_block(shard, 1, seqno);
   } else if (word == "byutime") {
@@ -912,21 +935,21 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress
                                     true);
   LOG(INFO) << "requesting account state for " << workchain << ":" << addr.to_hex() << " with respect to "
             << ref_blkid.to_str() << " with savefile `" << filename << "` and mode " << mode;
-  return envelope_send_query(
+  return envelope_send_query(std::move(b), [ Self = actor_id(this), workchain, addr, ref_blkid, filename,
-      std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode](td::Result<td::BufferSlice> R) {
+                                             mode ](td::Result<td::BufferSlice> R) {
-        if (R.is_error()) {
+    if (R.is_error()) {
-          return;
+      return;
-        }
+    }
-        auto F = ton::fetch_tl_object<ton::lite_api::liteServer_accountState>(R.move_as_ok(), true);
+    auto F = ton::fetch_tl_object<ton::lite_api::liteServer_accountState>(R.move_as_ok(), true);
-        if (F.is_error()) {
+    if (F.is_error()) {
-          LOG(ERROR) << "cannot parse answer to liteServer.getAccountState";
+      LOG(ERROR) << "cannot parse answer to liteServer.getAccountState";
-        } else {
+    } else {
-          auto f = F.move_as_ok();
+      auto f = F.move_as_ok();
-          td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_),
+      td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_),
-                                        ton::create_block_id(f->shardblk_), std::move(f->shard_proof_),
+                                    ton::create_block_id(f->shardblk_), std::move(f->shard_proof_),
-                                        std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode);
+                                    std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode);
-        }
+    }
-      });
+  });
 }
 
 bool TestNode::parse_run_method(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid,
@@ -951,22 +974,23 @@ bool TestNode::parse_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a
                                     true);
   LOG(INFO) << "requesting account state for " << workchain << ":" << addr.to_hex() << " with respect to "
             << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters";
-  return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name,
+  return envelope_send_query(
-                                            params = std::move(params)](td::Result<td::BufferSlice> R) mutable {
+      std::move(b), [ Self = actor_id(this), workchain, addr, ref_blkid, method_name,
-    if (R.is_error()) {
+                      params = std::move(params) ](td::Result<td::BufferSlice> R) mutable {
-      return;
+        if (R.is_error()) {
-    }
+          return;
-    auto F = ton::fetch_tl_object<ton::lite_api::liteServer_accountState>(R.move_as_ok(), true);
+        }
-    if (F.is_error()) {
+        auto F = ton::fetch_tl_object<ton::lite_api::liteServer_accountState>(R.move_as_ok(), true);
-      LOG(ERROR) << "cannot parse answer to liteServer.getAccountState";
+        if (F.is_error()) {
-    } else {
+          LOG(ERROR) << "cannot parse answer to liteServer.getAccountState";
-      auto f = F.move_as_ok();
+        } else {
-      td::actor::send_closure_later(Self, &TestNode::run_smc_method, ref_blkid, ton::create_block_id(f->id_),
+          auto f = F.move_as_ok();
-                                    ton::create_block_id(f->shardblk_), std::move(f->shard_proof_),
+          td::actor::send_closure_later(Self, &TestNode::run_smc_method, ref_blkid, ton::create_block_id(f->id_),
-                                    std::move(f->proof_), std::move(f->state_), workchain, addr, method_name,
+                                        ton::create_block_id(f->shardblk_), std::move(f->shard_proof_),
-                                    std::move(params));
+                                        std::move(f->proof_), std::move(f->state_), workchain, addr, method_name,
-    }
+                                        std::move(params));
-  });
+        }
+      });
 }
 
 bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workchain, ton::StdSmcAddress addr,
@@ -987,7 +1011,7 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc
   LOG(INFO) << "requesting transaction " << lt << " of " << workchain << ":" << addr.to_hex() << " from block "
             << blkid.to_str();
   return envelope_send_query(
-      std::move(b), [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result<td::BufferSlice> R) -> void {
+      std::move(b), [ Self = actor_id(this), workchain, addr, lt, blkid, dump ](td::Result<td::BufferSlice> R)->void {
         if (R.is_error()) {
           return;
         }
@@ -1013,7 +1037,7 @@ bool TestNode::get_last_transactions(ton::WorkchainId workchain, ton::StdSmcAddr
   LOG(INFO) << "requesting " << count << " last transactions from " << lt << ":" << hash.to_hex() << " of " << workchain
             << ":" << addr.to_hex();
   return envelope_send_query(
-      std::move(b), [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result<td::BufferSlice> R) {
+      std::move(b), [ Self = actor_id(this), workchain, addr, lt, hash, count, dump ](td::Result<td::BufferSlice> R) {
         if (R.is_error()) {
           return;
         }
@@ -1456,7 +1480,7 @@ bool TestNode::get_block_transactions(ton::BlockIdExt blkid, int mode, unsigned
                                     true);
   LOG(INFO) << "requesting " << count << " transactions from block " << blkid.to_str() << " starting from account "
             << acc_addr.to_hex() << " lt " << lt;
-  return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result<td::BufferSlice> R) {
+  return envelope_send_query(std::move(b), [ Self = actor_id(this), mode ](td::Result<td::BufferSlice> R) {
     if (R.is_error()) {
       return;
     }
@@ -1504,7 +1528,7 @@ bool TestNode::get_all_shards(bool use_last, ton::BlockIdExt blkid) {
   auto b = ton::serialize_tl_object(
       ton::create_tl_object<ton::lite_api::liteServer_getAllShardsInfo>(ton::create_tl_lite_block_id(blkid)), true);
   LOG(INFO) << "requesting recent shard configuration";
-  return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result<td::BufferSlice> R) -> void {
+  return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result<td::BufferSlice> R)->void {
     if (R.is_error()) {
       return;
     }
@@ -1585,8 +1609,8 @@ bool TestNode::get_config_params(ton::BlockIdExt blkid, int mode, std::string fi
                                           true);
   LOG(INFO) << "requesting " << params.size() << " configuration parameters with respect to masterchain block "
             << blkid.to_str();
-  return envelope_send_query(std::move(b), [Self = actor_id(this), mode, filename,
+  return envelope_send_query(std::move(b), [ Self = actor_id(this), mode, filename,
-                                            params = std::move(params)](td::Result<td::BufferSlice> R) mutable {
+                                             params = std::move(params) ](td::Result<td::BufferSlice> R) mutable {
     if (R.is_error()) {
       return;
     }
@@ -1683,7 +1707,7 @@ bool TestNode::get_block(ton::BlockIdExt blkid, bool dump) {
   auto b = ton::serialize_tl_object(
       ton::create_tl_object<ton::lite_api::liteServer_getBlock>(ton::create_tl_lite_block_id(blkid)), true);
   return envelope_send_query(
-      std::move(b), [Self = actor_id(this), blkid, dump](td::Result<td::BufferSlice> res) -> void {
+      std::move(b), [ Self = actor_id(this), blkid, dump ](td::Result<td::BufferSlice> res)->void {
         if (res.is_error()) {
           LOG(ERROR) << "cannot obtain block " << blkid.to_str()
                      << " from server : " << res.move_as_error().to_string();
@@ -1712,7 +1736,7 @@ bool TestNode::get_state(ton::BlockIdExt blkid, bool dump) {
   auto b = ton::serialize_tl_object(
       ton::create_tl_object<ton::lite_api::liteServer_getState>(ton::create_tl_lite_block_id(blkid)), true);
   return envelope_send_query(
-      std::move(b), [Self = actor_id(this), blkid, dump](td::Result<td::BufferSlice> res) -> void {
+      std::move(b), [ Self = actor_id(this), blkid, dump ](td::Result<td::BufferSlice> res)->void {
         if (res.is_error()) {
           LOG(ERROR) << "cannot obtain state " << blkid.to_str()
                      << " from server : " << res.move_as_error().to_string();
@@ -1852,7 +1876,7 @@ bool TestNode::get_block_header(ton::BlockIdExt blkid, int mode) {
   LOG(INFO) << "got block header request for " << blkid.to_str() << " with mode " << mode;
   auto b = ton::serialize_tl_object(
       ton::create_tl_object<ton::lite_api::liteServer_getBlockHeader>(ton::create_tl_lite_block_id(blkid), mode), true);
-  return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result<td::BufferSlice> res) -> void {
+  return envelope_send_query(std::move(b), [ Self = actor_id(this), blkid ](td::Result<td::BufferSlice> res)->void {
     if (res.is_error()) {
       LOG(ERROR) << "cannot obtain block header for " << blkid.to_str()
                  << " from server : " << res.move_as_error().to_string();
@@ -1883,7 +1907,7 @@ bool TestNode::lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg) {
                                         mode, ton::create_tl_lite_block_id_simple(id), arg, (td::uint32)arg),
                                     true);
   return envelope_send_query(
-      std::move(b), [Self = actor_id(this), id, mode, arg](td::Result<td::BufferSlice> res) -> void {
+      std::move(b), [ Self = actor_id(this), id, mode, arg ](td::Result<td::BufferSlice> res)->void {
         if (res.is_error()) {
           LOG(ERROR) << "cannot look up block header for " << id.to_str() << " with mode " << mode << " and argument "
                      << arg << " from server : " << res.move_as_error().to_string();
@@ -1901,7 +1925,6 @@ bool TestNode::lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg) {
           }
         }
       });
-  return false;
 }
 
 bool TestNode::show_block_header(ton::BlockIdExt blkid, Ref<vm::Cell> root, int mode) {
@@ -1979,6 +2002,135 @@ void TestNode::got_block_header(ton::BlockIdExt blkid, td::BufferSlice data, int
   show_new_blkids();
 }
 
+bool TestNode::get_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode) {
+  if (!(mode & 1)) {
+    to.invalidate_clear();
+  }
+  if (!(mode & 0x2000)) {
+    LOG(INFO) << "got block proof request from " << from.to_str() << " to "
+            << ((mode & 1) ? to.to_str() : "last masterchain block") << " with mode=" << mode;
+  } else {
+    LOG(DEBUG) << "got block proof request from " << from.to_str() << " to "
+            << ((mode & 1) ? to.to_str() : "last masterchain block") << " with mode=" << mode;
+  }
+  if (!from.is_masterchain_ext()) {
+    LOG(ERROR) << "source block " << from.to_str() << " is not a valid masterchain block id";
+    return false;
+  }
+  if ((mode & 1) && !to.is_masterchain_ext()) {
+    LOG(ERROR) << "destination block " << to.to_str() << " is not a valid masterchain block id";
+    return false;
+  }
+  auto b =
+      ton::serialize_tl_object(ton::create_tl_object<ton::lite_api::liteServer_getBlockProof>(
+                                   mode & 0xfff, ton::create_tl_lite_block_id(from), ton::create_tl_lite_block_id(to)),
+                               true);
+  return envelope_send_query(std::move(b), [ Self = actor_id(this), from, to, mode ](td::Result<td::BufferSlice> res) {
+    if (res.is_error()) {
+      LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block")
+                 << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string();
+    } else {
+      td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok());
+    }
+  });
+}
+
+td::Result<std::unique_ptr<block::BlockProofChain>> TestNode::deserialize_proof_chain(td::BufferSlice pchain) {
+  TRY_RESULT(f, ton::fetch_tl_object<ton::lite_api::liteServer_partialBlockProof>(std::move(pchain), true));
+  // deserialize proof chain
+  auto chain = std::make_unique<block::BlockProofChain>(ton::create_block_id(f->from_), ton::create_block_id(f->to_));
+  chain->complete = f->complete_;
+  for (auto& s : f->steps_) {
+    bool ok = false;
+    td::BufferSlice dest_proof, proof, state_proof;
+    ton::lite_api::downcast_call(
+        *s,
+        td::overloaded(
+            [&](ton::lite_api::liteServer_blockLinkBack& s) {
+              auto& link = chain->new_link(ton::create_block_id(s.from_), ton::create_block_id(s.to_), s.to_key_block_);
+              link.is_fwd = false;
+              // dest_proof:bytes state_proof:bytes proof:bytes
+              dest_proof = std::move(s.dest_proof_);
+              state_proof = std::move(s.state_proof_);
+              proof = std::move(s.proof_);
+              ok = true;
+            },
+            [&](ton::lite_api::liteServer_blockLinkForward& s) {
+              auto& link = chain->new_link(ton::create_block_id(s.from_), ton::create_block_id(s.to_), s.to_key_block_);
+              link.is_fwd = true;
+              // dest_proof:bytes config_proof:bytes signatures:liteServer.SignatureSet
+              dest_proof = std::move(s.dest_proof_);
+              proof = std::move(s.config_proof_);
+              link.cc_seqno = s.signatures_->catchain_seqno_;
+              link.validator_set_hash = s.signatures_->validator_set_hash_;
+              for (auto& sig : s.signatures_->signatures_) {
+                link.signatures.emplace_back(std::move(sig->node_id_short_), std::move(sig->signature_));
+              }
+              ok = true;
+            },
+            [&](auto& obj) {}));
+    if (!ok) {
+      return td::Status::Error("unknown constructor of liteServer.BlockLink");
+    }
+    auto& link = chain->last_link();
+    if (!dest_proof.empty()) {
+      auto d_res = vm::std_boc_deserialize(std::move(dest_proof));
+      if (d_res.is_error()) {
+        return td::Status::Error("cannot deserialize dest_proof in a block proof link: "s +
+                                 d_res.move_as_error().to_string());
+      }
+      link.dest_proof = d_res.move_as_ok();
+    }
+    auto d_res = vm::std_boc_deserialize(std::move(proof));
+    if (d_res.is_error()) {
+      return td::Status::Error("cannot deserialize proof in a block proof link: "s + d_res.move_as_error().to_string());
+    }
+    link.proof = d_res.move_as_ok();
+    if (!link.is_fwd) {
+      d_res = vm::std_boc_deserialize(std::move(state_proof));
+      if (d_res.is_error()) {
+        return td::Status::Error("cannot deserialize state_proof in a block proof link: "s +
+                                 d_res.move_as_error().to_string());
+      }
+      link.state_proof = d_res.move_as_ok();
+    }
+    LOG(DEBUG) << "deserialized a " << (link.is_fwd ? "forward" : "backward") << " BlkProofLink from "
+               << link.from.to_str() << " to " << link.to.to_str() << " with " << link.signatures.size()
+               << " signatures";
+  }
+  LOG(DEBUG) << "deserialized a BlkProofChain of " << chain->link_count() << " links";
+  return std::move(chain);
+}
+
+void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice pchain) {
+  LOG(INFO) << "got block proof from " << from.to_str() << " to "
+            << ((mode & 1) ? to.to_str() : "last masterchain block") << " with mode=" << mode << " (" << pchain.size()
+            << " bytes)";
+  auto res = deserialize_proof_chain(std::move(pchain));
+  if (res.is_error()) {
+    LOG(ERROR) << "cannot deserialize liteServer.partialBlockProof: " << res.move_as_error();
+    return;
+  }
+  auto chain = res.move_as_ok();
+  auto err = chain->validate();
+  if (err.is_error()) {
+    LOG(ERROR) << "block proof chain is invalid: " << err;
+    return;
+  }
+  LOG(INFO) << "valid " << (chain->complete ? "" : "in") << "complete proof chain: last block is " << chain->to.to_str()
+            << ", last key block is " << (chain->has_key_block ? chain->key_blkid.to_str() : "(undefined)");
+  // TODO: if `from` was a trusted key block, then mark `to` as a trusted key block, and update the known value of latest trusted key block if `to` is newer
+  if (!chain->complete && (mode & 0x1000)) {
+    get_block_proof(chain->to, to, mode | 0x2000);
+    return;
+  }
+  if (chain->has_key_block) {
+    register_blkid(chain->key_blkid);
+  }
+  register_blkid(chain->to);
+  show_new_blkids();
+}
+
 int main(int argc, char* argv[]) {
   SET_VERBOSITY_LEVEL(verbosity_INFO);
   td::set_default_failure_signal_handler();
@@ -2041,12 +2193,14 @@ int main(int argc, char* argv[]) {
     return td::Status::OK();
   });
   p.add_option('d', "daemonize", "set SIGHUP", [&]() {
-    td::set_signal_handler(td::SignalType::HangUp, [](int sig) {
+    td::set_signal_handler(td::SignalType::HangUp,
+                           [](int sig) {
 #if TD_DARWIN || TD_LINUX
-      close(0);
+                             close(0);
-      setsid();
+                             setsid();
 #endif
-    }).ensure();
+                           })
+        .ensure();
     return td::Status::OK();
   });
 #if TD_DARWIN || TD_LINUX

lite-client/lite-client.h

@@ -32,6 +32,7 @@
 #include "terminal/terminal.h"
 #include "vm/cells.h"
 #include "vm/stack.hpp"
+#include "block/block.h"
 #include "td/utils/filesystem.h"
 
 using td::Ref;
@@ -39,11 +40,13 @@ using td::Ref;
 class TestNode : public td::actor::Actor {
  private:
   std::string global_config_ = "ton-global.config";
-
+  static constexpr int min_ls_version = 0x101;         // server version >= 1.1
+  static constexpr long long min_ls_capabilities = 1;  // at least +1 = build proof chains
   td::actor::ActorOwn<ton::adnl::AdnlExtClient> client_;
   td::actor::ActorOwn<td::TerminalIO> io_;
 
   bool readline_enabled_ = true;
+  bool server_ok_ = false;
   td::int32 liteserver_idx_ = -1;
 
   bool ready_ = false;
@@ -138,6 +141,9 @@ class TestNode : public td::actor::Actor {
                               ton::LogicalTime lt);
   void got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete,
                               std::vector<TransId> trans, td::BufferSlice proof);
+  bool get_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode);
+  void got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice res);
+  td::Result<std::unique_ptr<block::BlockProofChain>> deserialize_proof_chain(td::BufferSlice pchain);
   bool do_parse_line();
   bool show_help(std::string command);
   std::string get_word(char delim = ' ');

rldp/rldp-peer.cpp

@@ -38,9 +38,9 @@ void RldpTransferSenderImpl::create_encoder() {
     return;
   }
   td::BufferSlice D = data_.clone();
-  D.confirm_read(part_ * slice_size());
+  D.confirm_read(td::narrow_cast<std::size_t>(part_ * slice_size()));
   if (D.size() > slice_size()) {
-    D.truncate(slice_size());
+    D.truncate(td::narrow_cast<std::size_t>(slice_size()));
   }
   fec_type_ = td::fec::RaptorQEncoder::Parameters{D.size(), symbol_size(), 0};
   auto E = fec_type_.create_encoder(std::move(D));
@@ -143,7 +143,7 @@ void RldpTransferReceiverImpl::receive_part(fec::FecType fec_type, td::uint32 pa
                                           << " data_size=" << data.data.size() << " part=" << part_));
         return;
       }
-      data_.as_slice().remove_prefix(offset_).copy_from(data.data.as_slice());
+      data_.as_slice().remove_prefix(td::narrow_cast<std::size_t>(offset_)).copy_from(data.data.as_slice());
       offset_ += data.data.size();
       auto obj = create_tl_object<ton_api::rldp_complete>(transfer_id_, part_);
       td::actor::send_closure(adnl_, &adnl::Adnl::send_message, local_id_, peer_id_, serialize_tl_object(obj, true));

rldp/rldp-peer.hpp

@@ -87,7 +87,7 @@ class RldpTransferReceiverImpl : public RldpTransferReceiver {
                     td::BufferSlice data) override;
   void alarm() override;
   void start_up() override {
-    data_ = td::BufferSlice(total_size_);
+    data_ = td::BufferSlice(td::narrow_cast<size_t>(total_size_));
   }
 
   RldpTransferReceiverImpl(TransferId transfer_id, adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id,

tdactor/td/actor/core/ActorInfoCreator.h

@@ -83,6 +83,9 @@ class ActorInfoCreator {
   ~ActorInfoCreator() {
     clear();
   }
+  void ensure_empty() {
+    pool_.for_each([](auto &actor_info) { LOG(ERROR) << actor_info.get_name(); });
+  }
 
  private:
   SharedObjectPool<ActorInfo> pool_;

tdactor/td/actor/core/Scheduler.cpp

@@ -282,6 +282,12 @@ void Scheduler::close_scheduler_group(SchedulerGroupInfo &group_info) {
       worker->actor_info_creator.clear();
     }
   }
+  //for (auto &scheduler : group_info.schedulers) {
+  //scheduler.io_worker->actor_info_creator.ensure_empty();
+  //for (auto &worker : scheduler.cpu_workers) {
+  //worker->actor_info_creator.ensure_empty();
+  //}
+  //}
 }
 }  // namespace core
 }  // namespace actor

tdutils/td/utils/port/StdStreams.cpp

@@ -27,6 +27,7 @@
 #include "td/utils/Slice.h"
 
 #include <atomic>
+#include <cassert>
 
 namespace td {
 
@@ -35,6 +36,7 @@ template <int id>
 static FileFd &get_file_fd() {
   static FileFd result = FileFd::from_native_fd(NativeFd(id, true));
   static auto guard = td::ScopeExit() + [&] { result.move_as_native_fd().release(); };
+  assert(!result.empty());
   return result;
 }
 

tdutils/td/utils/port/signals.cpp

@@ -329,6 +329,9 @@ static void default_failure_signal_handler(int sig) {
 }
 
 Status set_default_failure_signal_handler() {
+#if TD_PORT_POSIX
+  Stdin();  // init static variables before atexit
+#endif
   std::atexit(block_stdin);
   TRY_STATUS(setup_signals_alt_stack());
   TRY_STATUS(set_signal_handler(SignalType::Abort, default_failure_signal_handler));

terminal/terminal.cpp

@@ -34,7 +34,15 @@ void TerminalLogInterface::append(CSlice slice, int log_level) {
     default_log_interface->append(slice, log_level);
   } else {
     instance_->deactivate_readline();
-    td::TsCerr() << TC_GREEN << slice << TC_EMPTY;
+    std::string color;
+    if (log_level == 0 || log_level == 1) {
+      color = TC_RED;
+    } else if (log_level == 2) {
+      color = TC_YELLOW;
+    } else {
+      color = TC_GREEN;
+    }
+    td::TsCerr() << color << slice << TC_EMPTY;
     instance_->reactivate_readline();
     if (log_level == VERBOSITY_NAME(FATAL)) {
       process_fatal_error(slice);
@@ -106,13 +114,14 @@ void TerminalIOImpl::tear_down() {
   out_mutex_.lock();
 #ifdef USE_READLINE
   if (use_readline_) {
-    out_mutex_.lock();
+    //out_mutex_.lock();
     rl_callback_handler_remove();
-    out_mutex_.unlock();
+    //out_mutex_.unlock();
   }
 #endif
   instance_ = nullptr;
   out_mutex_.unlock();
+  log_interface_.release();  // TODO: actually release memory
 }
 
 /*void TerminalIOImpl::read_line() {

test/test-catchain.cpp

@@ -57,9 +57,6 @@ struct Node {
   ton::adnl::AdnlNodeIdFull adnl_id_full;
 };
 
-std::vector<Node> nodes;
-td::uint32 total_nodes = 11;
-
 class CatChainInst : public td::actor::Actor {
  public:
   class PayloadExtra : public ton::catchain::CatChainBlock::Extra {
@@ -212,6 +209,9 @@ class CatChainInst : public td::actor::Actor {
   std::vector<td::uint64> prev_values_;
 };
 
+static std::vector<Node> nodes;
+static td::uint32 total_nodes = 11;
+
 int main(int argc, char *argv[]) {
   SET_VERBOSITY_LEVEL(verbosity_INFO);
   td::set_default_failure_signal_handler().ensure();

test/test-validator-session-state.cpp

@@ -27,6 +27,7 @@
 */
 #include "adnl/adnl.h"
 
+#include "td/utils/misc.h"
 #include "td/utils/port/signals.h"
 #include "td/utils/port/path.h"
 #include "td/utils/Random.h"
@@ -34,6 +35,7 @@
 #include "validator-session/validator-session-description.h"
 #include "validator-session/validator-session-state.h"
 
+#include <limits>
 #include <memory>
 #include <set>
 
@@ -193,7 +195,8 @@ class Description : public ton::validatorsession::ValidatorSessionDescription {
 
   Description(ton::validatorsession::ValidatorSessionOptions opts, td::uint32 total_nodes)
       : opts_(opts), total_nodes_(total_nodes) {
-    pdata_size_[0] = 1ull << 33;
+    pdata_size_[0] =
+        static_cast<std::size_t>(std::numeric_limits<std::size_t>::max() < (1ull << 32) ? 1ull << 30 : 1ull << 33);
     pdata_size_[1] = 1 << 22;
     pdata_[0] = new td::uint8[pdata_size_[0]];
     pdata_[1] = new td::uint8[pdata_size_[1]];

tl/generate/scheme/lite_api.tl

@@ -43,8 +43,8 @@ liteServer.transactionId3 account:int256 lt:long = liteServer.TransactionId3;
 liteServer.blockTransactions id:tonNode.blockIdExt req_count:# incomplete:Bool ids:(vector liteServer.transactionId) proof:bytes = liteServer.BlockTransactions;
 liteServer.signature node_id_short:int256 signature:bytes = liteServer.Signature;
 liteServer.signatureSet validator_set_hash:int catchain_seqno:int signatures:(vector liteServer.signature) = liteServer.SignatureSet;
-liteServer.blockLinkBack to_key_block:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt dest_proof:bytes shard_proof:bytes proof:bytes = liteServer.BlockLink;
+liteServer.blockLinkBack to_key_block:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt dest_proof:bytes proof:bytes state_proof:bytes = liteServer.BlockLink;
-liteServer.blockLinkForward to_key_block:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt dest_proof:bytes config_proof:bytes signatures:liteServer.signatureSet = liteServer.BlockLink;
+liteServer.blockLinkForward to_key_block:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt dest_proof:bytes config_proof:bytes signatures:liteServer.SignatureSet = liteServer.BlockLink;
 liteServer.partialBlockProof complete:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt steps:(vector liteServer.BlockLink) = liteServer.PartialBlockProof;
 liteServer.configInfo mode:# id:tonNode.blockIdExt state_proof:bytes config_proof:bytes = liteServer.ConfigInfo;
 

ton/ton-types.h

@@ -310,6 +310,8 @@ struct ZeroStateIdExt {
 struct BlockSignature {
   NodeIdShort node;
   td::BufferSlice signature;
+  BlockSignature(const NodeIdShort& _node, td::BufferSlice _signature) : node(_node), signature(std::move(_signature)) {
+  }
 };
 
 struct ReceivedBlock {
@@ -356,6 +358,9 @@ struct Ed25519_PublicKey {
   operator Bits256() const {
     return _pubkey;
   }
+  td::Slice as_slice() const {
+    return _pubkey.as_slice();
+  }
   bool operator==(const Ed25519_PublicKey& other) const {
     return _pubkey == other._pubkey;
   }

tonlib/CMakeLists.txt

@@ -64,6 +64,9 @@ if (TONLIB_ENABLE_JNI AND NOT ANDROID) # jni is available by default on Android
   target_link_libraries(tonlib PUBLIC ${JAVA_JVM_LIBRARY})
 endif()
 
+add_executable(tonlib-cli tonlib/tonlib-cli.cpp)
+target_link_libraries(tonlib-cli tonlib tdactor tdutils terminal)
+
 if (NOT CMAKE_CROSSCOMPILING)
   if (TD_ENABLE_JNI)
 	  #FIXME

tonlib/tonlib/ExtClient.h

@@ -54,6 +54,7 @@ class ExtClient {
   template <class QueryT>
   void send_query(QueryT query, td::Promise<typename QueryT::ReturnType> promise) {
     auto raw_query = ton::serialize_tl_object(&query, true);
+    LOG(ERROR) << "send query to liteserver: " << to_string(query);
     td::BufferSlice liteserver_query =
         ton::serialize_tl_object(ton::create_tl_object<ton::lite_api::liteServer_query>(std::move(raw_query)), true);
 
@@ -65,7 +66,8 @@ class ExtClient {
           auto f = r_error.move_as_ok();
           return td::Status::Error(f->code_, f->message_);
         }
-        return ton::fetch_result<QueryT>(std::move(data));
+        auto res = ton::fetch_result<QueryT>(std::move(data));
+        return std::move(res);
       }());
     });
   }

tonlib/tonlib/LastBlock.cpp

@@ -43,6 +43,8 @@ void LastBlock::on_masterchain_info(
     auto info = r_info.move_as_ok();
     update_zero_state(create_zero_state_id(info->init_));
     update_mc_last_block(create_block_id(info->last_));
+  } else {
+    LOG(WARNING) << "Failed liteServer_getMasterchainInfo " << r_info.error();
   }
   for (auto& promise : promises_) {
     auto copy = mc_last_block_id_;

tonlib/tonlib/TonlibCallback.h

@@ -21,6 +21,7 @@
 #include "auto/tl/tonlib_api.h"
 
 namespace tonlib_api = ton::tonlib_api;
+namespace tonlib {
 
 class TonlibCallback {
  public:
@@ -28,3 +29,4 @@ class TonlibCallback {
   virtual void on_error(std::uint64_t id, tonlib_api::object_ptr<tonlib_api::error> error) = 0;
   virtual ~TonlibCallback() = default;
 };
+}  // namespace tonlib

tonlib/tonlib/TonlibClient.cpp

@@ -54,6 +54,10 @@ auto try_f(F&& f) noexcept -> decltype(f()) {
 
 #define TRY_VM(f) try_f([&] { return f; })
 
+tonlib_api::object_ptr<tonlib_api::error> status_to_tonlib_api(const td::Status& status) {
+  return tonlib_api::make_object<tonlib_api::error>(status.code(), status.message().str());
+}
+
 static block::AccountState create_account_state(ton::tl_object_ptr<ton::lite_api::liteServer_accountState> from) {
   block::AccountState res;
   res.blk = ton::create_block_id(from->id_);
@@ -177,7 +181,6 @@ class GetRawAccountState : public td::actor::Actor {
     auto account_state = create_account_state(std::move(raw_account_state));
     TRY_RESULT(info, account_state.validate(last_block_, address_));
     auto serialized_state = account_state.state.clone();
-    LOG(ERROR) << serialized_state.size();
     RawAccountState res;
     res.info = std::move(info);
     auto cell = res.info.root;
@@ -288,6 +291,7 @@ void TonlibClient::on_result(td::uint64 id, tonlib_api::object_ptr<tonlib_api::O
   callback_->on_result(id, std::move(response));
 }
 void TonlibClient::request(td::uint64 id, tonlib_api::object_ptr<tonlib_api::Function> function) {
+  LOG(ERROR) << to_string(function);
   if (function == nullptr) {
     LOG(ERROR) << "Receive empty static request";
     return on_result(id, tonlib_api::make_object<tonlib_api::error>(400, "Request is empty"));
@@ -311,7 +315,7 @@ void TonlibClient::request(td::uint64 id, tonlib_api::object_ptr<tonlib_api::Fun
     td::Promise<ReturnType> promise = [actor_id = actor_id(self), id](td::Result<ReturnType> r_result) {
       tonlib_api::object_ptr<tonlib_api::Object> result;
       if (r_result.is_error()) {
-        result = tonlib_api::make_object<tonlib_api::error>(r_result.error().code(), r_result.error().message().str());
+        result = status_to_tonlib_api(r_result.error());
       } else {
         result = r_result.move_as_ok();
       }
@@ -343,6 +347,9 @@ tonlib_api::object_ptr<tonlib_api::Object> TonlibClient::static_request(
 bool TonlibClient::is_static_request(td::int32 id) {
   switch (id) {
     case tonlib_api::runTests::ID:
+    case tonlib_api::raw_getAccountAddress::ID:
+    case tonlib_api::testWallet_getAccountAddress::ID:
+    case tonlib_api::testGiver_getAccountAddress::ID:
       return true;
     default:
       return false;
@@ -366,6 +373,37 @@ tonlib_api::object_ptr<tonlib_api::Object> TonlibClient::do_static_request(const
   runner.run_all();
   return tonlib_api::make_object<tonlib_api::ok>();
 }
+td::Result<block::StdAddress> get_account_address(const tonlib_api::raw_initialAccountState& raw_state) {
+  TRY_RESULT(code, vm::std_boc_deserialize(raw_state.code_));
+  TRY_RESULT(data, vm::std_boc_deserialize(raw_state.data_));
+  return GenericAccount::get_address(0 /*zerochain*/, GenericAccount::get_init_state(std::move(code), std::move(data)));
+}
+
+td::Result<block::StdAddress> get_account_address(const tonlib_api::testWallet_initialAccountState& test_wallet_state) {
+  auto key = td::Ed25519::PublicKey(td::SecureString(test_wallet_state.public_key_));
+  return GenericAccount::get_address(0 /*zerochain*/, TestWallet::get_init_state(key));
+}
+
+tonlib_api::object_ptr<tonlib_api::Object> TonlibClient::do_static_request(
+    const tonlib_api::raw_getAccountAddress& request) {
+  auto r_account_address = get_account_address(*request.initital_account_state_);
+  if (r_account_address.is_error()) {
+    return status_to_tonlib_api(r_account_address.error());
+  }
+  return tonlib_api::make_object<tonlib_api::accountAddress>(r_account_address.ok().rserialize());
+}
+tonlib_api::object_ptr<tonlib_api::Object> TonlibClient::do_static_request(
+    const tonlib_api::testWallet_getAccountAddress& request) {
+  auto r_account_address = get_account_address(*request.initital_account_state_);
+  if (r_account_address.is_error()) {
+    return status_to_tonlib_api(r_account_address.error());
+  }
+  return tonlib_api::make_object<tonlib_api::accountAddress>(r_account_address.ok().rserialize());
+}
+tonlib_api::object_ptr<tonlib_api::Object> TonlibClient::do_static_request(
+    const tonlib_api::testGiver_getAccountAddress& request) {
+  return tonlib_api::make_object<tonlib_api::accountAddress>(TestGiver::address().rserialize());
+}
 
 td::Status TonlibClient::do_request(const tonlib_api::init& request,
                                     td::Promise<object_ptr<tonlib_api::ok>>&& promise) {
@@ -411,17 +449,6 @@ td::Status TonlibClient::do_request(const tonlib_api::options_setConfig& request
   return td::Status::OK();
 }
 
-td::Result<block::StdAddress> get_account_address(const tonlib_api::raw_initialAccountState& raw_state) {
-  TRY_RESULT(code, vm::std_boc_deserialize(raw_state.code_));
-  TRY_RESULT(data, vm::std_boc_deserialize(raw_state.data_));
-  return GenericAccount::get_address(0 /*zerochain*/, GenericAccount::get_init_state(std::move(code), std::move(data)));
-}
-
-td::Result<block::StdAddress> get_account_address(const tonlib_api::testWallet_initialAccountState& test_wallet_state) {
-  auto key = td::Ed25519::PublicKey(td::SecureString(test_wallet_state.public_key_));
-  return GenericAccount::get_address(0 /*zerochain*/, TestWallet::get_init_state(key));
-}
-
 tonlib_api::object_ptr<tonlib_api::internal_transactionId> to_transaction_id(const block::AccountState::Info& info) {
   return tonlib_api::make_object<tonlib_api::internal_transactionId>(info.last_trans_lt,
                                                                      info.last_trans_hash.as_slice().str());
@@ -624,13 +651,6 @@ td::Result<tonlib_api::object_ptr<tonlib_api::generic_AccountState>> to_generic_
 
 // Raw
 
-td::Status TonlibClient::do_request(const tonlib_api::raw_getAccountAddress& request,
-                                    td::Promise<object_ptr<tonlib_api::accountAddress>>&& promise) {
-  TRY_RESULT(account_address, get_account_address(*request.initital_account_state_));
-  promise.set_value(tonlib_api::make_object<tonlib_api::accountAddress>(account_address.rserialize()));
-  return td::Status::OK();
-}
-
 td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessage& request,
                                     td::Promise<object_ptr<tonlib_api::ok>>&& promise) {
   td::Ref<vm::Cell> init_state;
@@ -701,12 +721,6 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactions& request,
   return td::Status::OK();
 }
 
-td::Status TonlibClient::do_request(tonlib_api::testGiver_getAccountAddress& request,
-                                    td::Promise<object_ptr<tonlib_api::accountAddress>>&& promise) {
-  promise.set_value(tonlib_api::make_object<tonlib_api::accountAddress>(TestGiver::address().rserialize()));
-  return td::Status::OK();
-}
-
 td::Result<KeyStorage::InputKey> from_tonlib(tonlib_api::inputKey& input_key) {
   if (!input_key.key_) {
     return td::Status::Error(400, "Field key must not be empty");
@@ -734,13 +748,6 @@ td::Status TonlibClient::do_request(const tonlib_api::testWallet_init& request,
       std::move(promise));
 }
 
-td::Status TonlibClient::do_request(const tonlib_api::testWallet_getAccountAddress& request,
-                                    td::Promise<object_ptr<tonlib_api::accountAddress>>&& promise) {
-  TRY_RESULT(account_address, get_account_address(*request.initital_account_state_));
-  promise.set_value(tonlib_api::make_object<tonlib_api::accountAddress>(account_address.rserialize()));
-  return td::Status::OK();
-}
-
 td::Status TonlibClient::do_request(const tonlib_api::testWallet_sendGrams& request,
                                     td::Promise<object_ptr<tonlib_api::ok>>&& promise) {
   if (!request.destination_) {
@@ -837,6 +844,7 @@ td::Status TonlibClient::do_request(const tonlib_api::generic_getAccountState& r
 td::Status TonlibClient::do_request(tonlib_api::generic_sendGrams& request,
                                     td::Promise<object_ptr<tonlib_api::ok>>&& promise) {
   TRY_RESULT(account_address, block::StdAddress::parse(request.source_->account_address_));
+  LOG(INFO) << "Send " << request.amount_ << " nanograms from " << account_address.rserialize();
   td::actor::create_actor<GetRawAccountState>(
       "GetAccountState", client_.get_client(), std::move(account_address),
       [promise = std::move(promise), self = this, actor_id = td::actor::actor_id(),
@@ -858,6 +866,7 @@ td::Status TonlibClient::do_request(tonlib_api::generic_sendGrams& request,
                                                    query = tonlib_api::testGiver_sendGrams(
                                                        std::move(destination), test_giver_state.account_state_->seqno_,
                                                        amount)]() mutable {
+                              LOG(INFO) << "Send " << to_string(query);
                               auto status = self->do_request(query, std::move(promise));
                               if (status.is_error()) {
                                 CHECK(promise);

tonlib/tonlib/TonlibClient.h

@@ -76,6 +76,9 @@ class TonlibClient : public td::actor::Actor {
     return tonlib_api::make_object<tonlib_api::error>(400, "Function can't be executed synchronously");
   }
   static object_ptr<tonlib_api::Object> do_static_request(const tonlib_api::runTests& request);
+  static object_ptr<tonlib_api::Object> do_static_request(const tonlib_api::raw_getAccountAddress& request);
+  static object_ptr<tonlib_api::Object> do_static_request(const tonlib_api::testWallet_getAccountAddress& request);
+  static object_ptr<tonlib_api::Object> do_static_request(const tonlib_api::testGiver_getAccountAddress& request);
   template <class T, class P>
   td::Status do_request(const T& request, P&& promise) {
     return td::Status::Error(400, "Function is unsupported");
@@ -88,8 +91,6 @@ class TonlibClient : public td::actor::Actor {
   td::Status do_request(const tonlib_api::options_setConfig& request,
                         td::Promise<object_ptr<tonlib_api::ok>>&& promise);
 
-  td::Status do_request(const tonlib_api::raw_getAccountAddress& request,
-                        td::Promise<object_ptr<tonlib_api::accountAddress>>&& promise);
   td::Status do_request(const tonlib_api::raw_sendMessage& request, td::Promise<object_ptr<tonlib_api::ok>>&& promise);
   td::Status do_request(tonlib_api::raw_getAccountState& request,
                         td::Promise<object_ptr<tonlib_api::raw_accountState>>&& promise);
@@ -97,8 +98,6 @@ class TonlibClient : public td::actor::Actor {
                         td::Promise<object_ptr<tonlib_api::raw_transactions>>&& promise);
 
   td::Status do_request(const tonlib_api::testWallet_init& request, td::Promise<object_ptr<tonlib_api::ok>>&& promise);
-  td::Status do_request(const tonlib_api::testWallet_getAccountAddress& request,
-                        td::Promise<object_ptr<tonlib_api::accountAddress>>&& promise);
   td::Status do_request(const tonlib_api::testWallet_sendGrams& request,
                         td::Promise<object_ptr<tonlib_api::ok>>&& promise);
   td::Status do_request(tonlib_api::testWallet_getAccountState& request,
@@ -106,8 +105,6 @@ class TonlibClient : public td::actor::Actor {
 
   td::Status do_request(const tonlib_api::testGiver_getAccountState& request,
                         td::Promise<object_ptr<tonlib_api::testGiver_accountState>>&& promise);
-  td::Status do_request(tonlib_api::testGiver_getAccountAddress& request,
-                        td::Promise<object_ptr<tonlib_api::accountAddress>>&& promise);
   td::Status do_request(const tonlib_api::testGiver_sendGrams& request,
                         td::Promise<object_ptr<tonlib_api::ok>>&& promise);
 

validator-engine/validator-engine.cpp

@@ -968,6 +968,15 @@ td::Status ValidatorEngine::load_global_config() {
   if (sync_ttl_ != 0) {
     validator_options_.write().set_sync_blocks_before(sync_ttl_);
   }
+  if (archive_ttl_ != 0) {
+    validator_options_.write().set_archive_ttl(archive_ttl_);
+  }
+  if (key_proof_ttl_ != 0) {
+    validator_options_.write().set_key_proof_ttl(key_proof_ttl_);
+  }
+  if (db_depth_ <= 32) {
+    validator_options_.write().set_filedb_depth(db_depth_);
+  }
 
   std::vector<ton::BlockIdExt> h;
   for (auto &x : conf.validator_->hardforks_) {
@@ -2775,6 +2784,15 @@ int main(int argc, char *argv[]) {
     acts.push_back([&x, fname = fname.str()]() { td::actor::send_closure(x, &ValidatorEngine::set_fift_dir, fname); });
     return td::Status::OK();
   });
+  p.add_option('F', "filedb-depth",
+               "depth of autodirs for blocks, proofs, etc. Default value is 2. You need to clear the "
+               "database, if you need to change this option",
+               [&](td::Slice fname) {
+                 acts.push_back([&x, fname = fname.str()]() {
+                   td::actor::send_closure(x, &ValidatorEngine::set_db_depth, td::to_integer<td::uint32>(fname));
+                 });
+                 return td::Status::OK();
+               });
   p.add_option('d', "daemonize", "set SIGHUP", [&]() {
 #if TD_DARWIN || TD_LINUX
     close(0);
@@ -2799,6 +2817,18 @@ int main(int argc, char *argv[]) {
                  acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_block_ttl, v); });
                  return td::Status::OK();
                });
+  p.add_option('A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400",
+               [&](td::Slice fname) {
+                 auto v = td::to_double(fname);
+                 acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_ttl, v); });
+                 return td::Status::OK();
+               });
+  p.add_option('K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10",
+               [&](td::Slice fname) {
+                 auto v = td::to_double(fname);
+                 acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_key_proof_ttl, v); });
+                 return td::Status::OK();
+               });
   p.add_option('S', "sync-before", "in initial sync download all blocks for last given seconds default=3600",
                [&](td::Slice fname) {
                  auto v = td::to_double(fname);

validator-engine/validator-engine.hpp

@@ -181,6 +181,9 @@ class ValidatorEngine : public td::actor::Actor {
   td::Clocks::Duration state_ttl_ = 0;
   td::Clocks::Duration block_ttl_ = 0;
   td::Clocks::Duration sync_ttl_ = 0;
+  td::Clocks::Duration archive_ttl_ = 0;
+  td::Clocks::Duration key_proof_ttl_ = 0;
+  td::uint32 db_depth_ = 33;
   bool read_config_ = false;
   bool started_keyring_ = false;
   bool started_ = false;
@@ -196,6 +199,9 @@ class ValidatorEngine : public td::actor::Actor {
     fift_dir_ = str;
   }
   void set_db_root(std::string db_root);
+  void set_db_depth(td::uint32 value) {
+    db_depth_ = value;
+  }
   void set_state_ttl(td::Clocks::Duration t) {
     state_ttl_ = t;
   }
@@ -205,6 +211,12 @@ class ValidatorEngine : public td::actor::Actor {
   void set_sync_ttl(td::Clocks::Duration t) {
     sync_ttl_ = t;
   }
+  void set_archive_ttl(td::Clocks::Duration t) {
+    archive_ttl_ = t;
+  }
+  void set_key_proof_ttl(td::Clocks::Duration t) {
+    key_proof_ttl_ = t;
+  }
   void add_ip(td::IPAddress addr) {
     addrs_.push_back(addr);
   }

validator/db/archiver.cpp

@@ -45,7 +45,8 @@ void BlockArchiver::got_block_handle(BlockHandle handle) {
     return;
   }
 
-  if (!handle_->is_applied() && !handle_->is_archived()) {
+  if (!handle_->is_applied() && !handle_->is_archived() &&
+      (!handle_->inited_is_key_block() || !handle_->is_key_block())) {
     // no need for this block
     // probably this block not in final chain
     // this will eventually delete all associated data

validator/db/celldb.cpp

@@ -142,7 +142,9 @@ void CellDbIn::gc() {
 }
 
 void CellDbIn::gc_cont(BlockHandle handle) {
-  CHECK(handle->inited_state_boc());
+  if (!handle->inited_state_boc()) {
+    LOG(WARNING) << "inited_state_boc=false, but state in db. blockid=" << handle->id();
+  }
   handle->set_deleted_state_boc();
 
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Unit> R) {

validator/db/filedb.cpp

@@ -34,11 +34,14 @@ std::string FileDb::get_file_name(const RefId& ref_id, bool create_dirs) {
   auto ref_id_hash = get_ref_id_hash(ref_id);
 
   auto s = ref_id_hash.to_hex();
-  if (create_dirs) {
+  std::string path = root_path_ + "/files/";
-    td::mkdir(root_path_ + "/files/" + s[0] + s[1] + "/").ensure();
+  for (td::uint32 i = 0; i < depth_; i++) {
-    td::mkdir(root_path_ + "/files/" + s[0] + s[1] + "/" + s[2] + s[3] + "/").ensure();
+    path = path + s[2 * i] + s[2 * i + 1] + "/";
+    if (create_dirs) {
+      td::mkdir(path).ensure();
+    }
   }
-  return root_path_ + "/files/" + s[0] + s[1] + "/" + s[2] + s[3] + "/" + s;
+  return path + s;
 }
 
 void FileDb::store_file(RefId ref_id, td::BufferSlice data, td::Promise<FileHash> promise) {
@@ -169,8 +172,8 @@ void FileDb::set_block(const RefIdHash& ref, DbEntry entry) {
   kv_->set(get_key(ref), entry.release()).ensure();
 }
 
-FileDb::FileDb(td::actor::ActorId<RootDb> root_db, std::string root_path, bool is_archive)
+FileDb::FileDb(td::actor::ActorId<RootDb> root_db, std::string root_path, td::uint32 depth, bool is_archive)
-    : root_db_(root_db), root_path_(root_path), is_archive_(is_archive) {
+    : root_db_(root_db), root_path_(root_path), depth_(depth), is_archive_(is_archive) {
 }
 
 void FileDb::start_up() {

validator/db/filedb.hpp

@@ -152,7 +152,7 @@ class FileDb : public td::actor::Actor {
   void gc();
   void skip_gc();
 
-  FileDb(td::actor::ActorId<RootDb> root_db, std::string root_path, bool is_archive);
+  FileDb(td::actor::ActorId<RootDb> root_db, std::string root_path, td::uint32 depth, bool is_archive);
 
  private:
   struct DbEntry {
@@ -187,6 +187,7 @@ class FileDb : public td::actor::Actor {
 
   std::string root_path_;
   std::string db_path_;
+  td::uint32 depth_;
 
   bool is_archive_;
 

validator/db/rootdb.cpp

@@ -408,8 +408,9 @@ void RootDb::get_async_serializer_state(td::Promise<AsyncSerializerState> promis
 void RootDb::start_up() {
   cell_db_ = td::actor::create_actor<CellDb>("celldb", actor_id(this), root_path_ + "/celldb/");
   block_db_ = td::actor::create_actor<BlockDb>("blockdb", actor_id(this), root_path_ + "/blockdb/");
-  file_db_ = td::actor::create_actor<FileDb>("filedb", actor_id(this), root_path_ + "/files/", false);
+  file_db_ = td::actor::create_actor<FileDb>("filedb", actor_id(this), root_path_ + "/files/", depth_, false);
-  archive_db_ = td::actor::create_actor<FileDb>("filedbarchive", actor_id(this), root_path_ + "/archive/", true);
+  archive_db_ =
+      td::actor::create_actor<FileDb>("filedbarchive", actor_id(this), root_path_ + "/archive/", depth_, true);
   lt_db_ = td::actor::create_actor<LtDb>("ltdb", actor_id(this), root_path_ + "/ltdb/");
   state_db_ = td::actor::create_actor<StateDb>("statedb", actor_id(this), root_path_ + "/state/");
   static_files_db_ = td::actor::create_actor<StaticFilesDb>("staticfilesdb", actor_id(this), root_path_ + "/static/");

validator/db/rootdb.hpp

@@ -36,8 +36,8 @@ namespace validator {
 class RootDb : public Db {
  public:
   enum class Flags : td::uint32 { f_started = 1, f_ready = 2, f_switched = 4, f_archived = 8 };
-  RootDb(td::actor::ActorId<ValidatorManager> validator_manager, std::string root_path)
+  RootDb(td::actor::ActorId<ValidatorManager> validator_manager, std::string root_path, td::uint32 depth)
-      : validator_manager_(validator_manager), root_path_(std::move(root_path)) {
+      : validator_manager_(validator_manager), root_path_(std::move(root_path)), depth_(depth) {
   }
 
   void start_up() override;
@@ -112,6 +112,7 @@ class RootDb : public Db {
   td::actor::ActorId<ValidatorManager> validator_manager_;
 
   std::string root_path_;
+  td::uint32 depth_;
 
   td::actor::ActorOwn<CellDb> cell_db_;
   td::actor::ActorOwn<BlockDb> block_db_;

validator/fabric.h

@@ -25,7 +25,8 @@ namespace ton {
 
 namespace validator {
 
-td::actor::ActorOwn<Db> create_db_actor(td::actor::ActorId<ValidatorManager> manager, std::string db_root_);
+td::actor::ActorOwn<Db> create_db_actor(td::actor::ActorId<ValidatorManager> manager, std::string db_root_,
+                                        td::uint32 depth);
 
 td::Result<td::Ref<BlockData>> create_block(BlockIdExt block_id, td::BufferSlice data);
 td::Result<td::Ref<BlockData>> create_block(ReceivedBlock data);

validator/impl/accept-block.cpp

@@ -671,7 +671,8 @@ void AcceptBlockQuery::got_proof_link(BlockIdExt id, Ref<ProofLink> proof) {
     // first link in chain
     if (ancestors_.size() != link_prev_.size() || ancestors_[0]->blk_ != link_prev_[0] ||
         (ancestors_.size() == 2 && ancestors_[1]->blk_ != link_prev_[1])) {
-      fatal_error("invalid first link at block "s + id.to_str() + " for shardchain block " + id_.to_str());
+      fatal_error("invalid first link at block "s + id.to_str() + " for shardchain block " + id_.to_str(),
+                  ErrorCode::cancelled);
       return;
     }
     create_topshard_blk_descr();
@@ -680,7 +681,8 @@ void AcceptBlockQuery::got_proof_link(BlockIdExt id, Ref<ProofLink> proof) {
     // intermediate link
     if (link_prev_.size() != 1 || ton::ShardIdFull(link_prev_[0].id) != ton::ShardIdFull(id_) ||
         link_prev_[0].id.seqno + 1 != id.id.seqno) {
-      fatal_error("invalid intermediate link at block "s + id.to_str() + " for shardchain block " + id_.to_str());
+      fatal_error("invalid intermediate link at block "s + id.to_str() + " for shardchain block " + id_.to_str(),
+                  ErrorCode::cancelled);
       return;
     }
     require_proof_link(link_prev_[0]);

validator/impl/fabric.cpp

@@ -38,8 +38,9 @@ namespace ton {
 
 namespace validator {
 
-td::actor::ActorOwn<Db> create_db_actor(td::actor::ActorId<ValidatorManager> manager, std::string db_root_) {
+td::actor::ActorOwn<Db> create_db_actor(td::actor::ActorId<ValidatorManager> manager, std::string db_root_,
-  return td::actor::create_actor<RootDb>("db", manager, db_root_);
+                                        td::uint32 depth) {
+  return td::actor::create_actor<RootDb>("db", manager, db_root_, depth);
 }
 
 td::Result<td::Ref<BlockData>> create_block(BlockIdExt block_id, td::BufferSlice data) {

validator/impl/liteserver.cpp

@@ -34,6 +34,8 @@
 #include "vm/dict.h"
 #include "vm/cells/MerkleProof.h"
 #include "shard.hpp"
+#include "validator-set.hpp"
+#include "signature-set.hpp"
 #include "fabric.h"
 #include <ctime>
 
@@ -66,6 +68,14 @@ void LiteQuery::abort_query(td::Status reason) {
   stop();
 }
 
+void LiteQuery::abort_query_ext(td::Status reason, std::string comment) {
+  LOG(INFO) << "aborted liteserver query: " << comment << " : " << reason.to_string();
+  if (promise_) {
+    promise_.set_error(reason.move_as_error_prefix(comment + " : "));
+  }
+  stop();
+}
+
 bool LiteQuery::fatal_error(td::Status error) {
   abort_query(std::move(error));
   return false;
@@ -167,7 +177,7 @@ void LiteQuery::perform_getTime() {
 void LiteQuery::perform_getVersion() {
   LOG(INFO) << "started a getVersion() liteserver query";
   td::int32 now = static_cast<td::int32>(std::time(nullptr));
-  auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_version>(0, 0x100, 0, now);
+  auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_version>(0, ls_version, ls_capabilities, now);
   finish_query(std::move(b));
 }
 
@@ -279,11 +289,10 @@ void LiteQuery::continue_getBlockHeader(BlockIdExt blkid, int mode, Ref<ton::val
   // create block header proof
   RootHash rhash{block_root->get_hash().bits()};
   CHECK(rhash == blkid.root_hash);
-  auto usage_tree = std::make_shared<vm::CellUsageTree>();
+  vm::MerkleProofBuilder mpb{block_root};
-  auto usage_cell = vm::UsageCell::create(block_root, usage_tree->root_ptr());
   block::gen::Block::Record blk;
   block::gen::BlockInfo::Record info;
-  if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(blk.info, info))) {
+  if (!(tlb::unpack_cell(mpb.root(), blk) && tlb::unpack_cell(blk.info, info))) {
     fatal_error("cannot unpack block header");
     return;
   }
@@ -327,12 +336,7 @@ void LiteQuery::continue_getBlockHeader(BlockIdExt blkid, int mode, Ref<ton::val
       }
     }
   }
-  auto proof = vm::MerkleProof::generate(std::move(block_root), usage_tree.get());
+  auto proof_data = mpb.extract_proof_boc();
-  if (proof.is_null()) {
-    fatal_error("cannot create Merkle proof for block header");
-    return;
-  }
-  auto proof_data = vm::std_boc_serialize(std::move(proof));
   if (proof_data.is_error()) {
     fatal_error(proof_data.move_as_error());
     return;
@@ -353,15 +357,27 @@ void LiteQuery::perform_getState(BlockIdExt blkid) {
     fatal_error("cannot request total state: possibly too large");
     return;
   }
-  td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
+  if (blkid.id.seqno) {
-                                [ Self = actor_id(this), blkid ](td::Result<Ref<ton::validator::ShardState>> res) {
+    td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
-                                  if (res.is_error()) {
+                                  [ Self = actor_id(this), blkid ](td::Result<Ref<ton::validator::ShardState>> res) {
-                                    td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
+                                    if (res.is_error()) {
-                                  } else {
+                                      td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
-                                    td::actor::send_closure_later(Self, &LiteQuery::continue_getState, blkid,
+                                    } else {
-                                                                  res.move_as_ok());
+                                      td::actor::send_closure_later(Self, &LiteQuery::continue_getState, blkid,
-                                  }
+                                                                    res.move_as_ok());
-                                });
+                                    }
+                                  });
+  } else {
+    td::actor::send_closure_later(manager_, &ValidatorManager::get_zero_state, blkid,
+                                  [ Self = actor_id(this), blkid ](td::Result<td::BufferSlice> res) {
+                                    if (res.is_error()) {
+                                      td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
+                                    } else {
+                                      td::actor::send_closure_later(Self, &LiteQuery::continue_getZeroState, blkid,
+                                                                    res.move_as_ok());
+                                    }
+                                  });
+  }
 }
 
 void LiteQuery::continue_getState(BlockIdExt blkid, Ref<ton::validator::ShardState> state) {
@@ -380,6 +396,14 @@ void LiteQuery::continue_getState(BlockIdExt blkid, Ref<ton::validator::ShardSta
   finish_query(std::move(b));
 }
 
+void LiteQuery::continue_getZeroState(BlockIdExt blkid, td::BufferSlice state) {
+  LOG(INFO) << "obtained data for getZeroState(" << blkid.to_str() << ")";
+  CHECK(!state.empty());
+  auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_blockState>(
+      ton::create_tl_lite_block_id(blkid), blkid.root_hash, blkid.file_hash, std::move(state));
+  finish_query(std::move(b));
+}
+
 void LiteQuery::perform_sendMessage(td::BufferSlice data) {
   LOG(INFO) << "started a sendMessage(<" << data.size() << " bytes>) liteserver query";
   auto res = ton::validator::create_ext_message(std::move(data));
@@ -402,15 +426,42 @@ bool LiteQuery::request_mc_block_data(BlockIdExt blkid) {
   }
   base_blk_id_ = blkid;
   ++pending_;
-  td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
+  td::actor::send_closure_later(
-                                [ Self = actor_id(this), blkid ](td::Result<Ref<BlockData>> res) {
+      manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
-                                  if (res.is_error()) {
+      [ Self = actor_id(this), blkid ](td::Result<Ref<BlockData>> res) {
-                                    td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
+        if (res.is_error()) {
-                                  } else {
+          td::actor::send_closure(Self, &LiteQuery::abort_query,
-                                    td::actor::send_closure_later(Self, &LiteQuery::got_mc_block_data, blkid,
+                                  res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : "));
-                                                                  res.move_as_ok());
+        } else {
-                                  }
+          td::actor::send_closure_later(Self, &LiteQuery::got_mc_block_data, blkid, res.move_as_ok());
-                                });
+        }
+      });
+  return true;
+}
+
+bool LiteQuery::request_mc_proof(BlockIdExt blkid, int mode) {
+  if (!blkid.is_masterchain() || !blkid.is_valid_full()) {
+    return fatal_error("reference block must belong to the masterchain");
+  }
+  if (!cont_set_) {
+    return fatal_error("continuation not set");
+  }
+  if (mode) {
+    base_blk_id_alt_ = blkid;
+  } else {
+    base_blk_id_ = blkid;
+  }
+  ++pending_;
+  td::actor::send_closure_later(
+      manager_, &ValidatorManager::get_block_proof_from_db_short, blkid,
+      [ Self = actor_id(this), blkid, mode ](td::Result<Ref<Proof>> res) {
+        if (res.is_error()) {
+          td::actor::send_closure(Self, &LiteQuery::abort_query,
+                                  res.move_as_error_prefix("cannot load proof for "s + blkid.to_str() + " : "));
+        } else {
+          td::actor::send_closure_later(Self, &LiteQuery::got_mc_block_proof, blkid, mode, res.move_as_ok());
+        }
+      });
   return true;
 }
 
@@ -423,15 +474,16 @@ bool LiteQuery::request_mc_block_state(BlockIdExt blkid) {
   }
   base_blk_id_ = blkid;
   ++pending_;
-  td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
+  td::actor::send_closure_later(
-                                [ Self = actor_id(this), blkid ](td::Result<Ref<ShardState>> res) {
+      manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
-                                  if (res.is_error()) {
+      [ Self = actor_id(this), blkid ](td::Result<Ref<ShardState>> res) {
-                                    td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
+        if (res.is_error()) {
-                                  } else {
+          td::actor::send_closure(Self, &LiteQuery::abort_query,
-                                    td::actor::send_closure_later(Self, &LiteQuery::got_mc_block_state, blkid,
+                                  res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : "));
-                                                                  res.move_as_ok());
+        } else {
-                                  }
+          td::actor::send_closure_later(Self, &LiteQuery::got_mc_block_state, blkid, res.move_as_ok());
-                                });
+        }
+      });
   return true;
 }
 
@@ -453,15 +505,16 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) {
   }
   blk_id_ = blkid;
   ++pending_;
-  td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
+  td::actor::send_closure_later(
-                                [ Self = actor_id(this), blkid ](td::Result<Ref<ShardState>> res) {
+      manager_, &ValidatorManager::get_shard_state_from_db_short, blkid,
-                                  if (res.is_error()) {
+      [ Self = actor_id(this), blkid ](td::Result<Ref<ShardState>> res) {
-                                    td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
+        if (res.is_error()) {
-                                  } else {
+          td::actor::send_closure(Self, &LiteQuery::abort_query,
-                                    td::actor::send_closure_later(Self, &LiteQuery::got_block_state, blkid,
+                                  res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : "));
-                                                                  res.move_as_ok());
+        } else {
-                                  }
+          td::actor::send_closure_later(Self, &LiteQuery::got_block_state, blkid, res.move_as_ok());
-                                });
+        }
+      });
   return true;
 }
 
@@ -474,15 +527,63 @@ bool LiteQuery::request_block_data(BlockIdExt blkid) {
   }
   blk_id_ = blkid;
   ++pending_;
-  td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
+  td::actor::send_closure_later(
-                                [ Self = actor_id(this), blkid ](td::Result<Ref<BlockData>> res) {
+      manager_, &ValidatorManager::get_block_data_from_db_short, blkid,
-                                  if (res.is_error()) {
+      [ Self = actor_id(this), blkid ](td::Result<Ref<BlockData>> res) {
-                                    td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
+        if (res.is_error()) {
-                                  } else {
+          td::actor::send_closure(Self, &LiteQuery::abort_query,
-                                    td::actor::send_closure_later(Self, &LiteQuery::got_block_data, blkid,
+                                  res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : "));
-                                                                  res.move_as_ok());
+        } else {
-                                  }
+          td::actor::send_closure_later(Self, &LiteQuery::got_block_data, blkid, res.move_as_ok());
-                                });
+        }
+      });
+  return true;
+}
+
+bool LiteQuery::request_proof_link(BlockIdExt blkid) {
+  if (!blkid.is_valid_full()) {
+    return fatal_error("invalid block id requested");
+  }
+  if (!cont_set_) {
+    return fatal_error("continuation not set");
+  }
+  blk_id_ = blkid;
+  ++pending_;
+  td::actor::send_closure_later(
+      manager_, &ValidatorManager::get_block_proof_link_from_db_short, blkid,
+      [ Self = actor_id(this), blkid ](td::Result<Ref<ProofLink>> res) {
+        if (res.is_error()) {
+          td::actor::send_closure(Self, &LiteQuery::abort_query,
+                                  res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : "));
+        } else {
+          td::actor::send_closure_later(Self, &LiteQuery::got_block_proof_link, blkid, res.move_as_ok());
+        }
+      });
+  return true;
+}
+
+bool LiteQuery::request_zero_state(BlockIdExt blkid) {
+  if (!blkid.is_valid_full()) {
+    return fatal_error("invalid block id requested");
+  }
+  if (blkid.seqno()) {
+    return fatal_error("invalid zerostate requested");
+  }
+  if (!cont_set_) {
+    return fatal_error("continuation not set");
+  }
+  blk_id_ = blkid;
+  ++pending_;
+  td::actor::send_closure_later(
+      manager_, &ValidatorManager::get_zero_state, blkid,
+      [ Self = actor_id(this), blkid ](td::Result<td::BufferSlice> res) {
+        if (res.is_error()) {
+          td::actor::send_closure(Self, &LiteQuery::abort_query,
+                                  res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : "));
+        } else {
+          td::actor::send_closure_later(Self, &LiteQuery::got_zero_state, blkid, res.move_as_ok());
+        }
+      });
   return true;
 }
 
@@ -595,6 +696,38 @@ void LiteQuery::got_mc_block_data(BlockIdExt blkid, Ref<BlockData> data) {
   dec_pending();
 }
 
+void LiteQuery::got_mc_block_proof(BlockIdExt blkid, int mode, Ref<Proof> proof) {
+  LOG(INFO) << "obtained data for getBlockProof(" << blkid.to_str() << ") needed by a liteserver query";
+  CHECK(proof.not_null());
+  if (mode) {
+    mc_proof_alt_ = Ref<ProofQ>(std::move(proof));
+    CHECK(mc_proof_alt_.not_null());
+    CHECK(blkid == base_blk_id_alt_);
+  } else {
+    mc_proof_ = Ref<ProofQ>(std::move(proof));
+    CHECK(mc_proof_.not_null());
+    CHECK(blkid == base_blk_id_);
+  }
+  dec_pending();
+}
+
+void LiteQuery::got_block_proof_link(BlockIdExt blkid, Ref<ProofLink> proof_link) {
+  LOG(INFO) << "obtained data for getBlockProofLink(" << blkid.to_str() << ") needed by a liteserver query";
+  CHECK(proof_link.not_null());
+  proof_link_ = Ref<ProofLinkQ>(std::move(proof_link));
+  CHECK(proof_link_.not_null());
+  CHECK(blkid == blk_id_);
+  dec_pending();
+}
+
+void LiteQuery::got_zero_state(BlockIdExt blkid, td::BufferSlice zerostate) {
+  LOG(INFO) << "obtained data for getZeroState(" << blkid.to_str() << ") needed by a liteserver query";
+  CHECK(!zerostate.empty());
+  buffer_ = std::move(zerostate);
+  CHECK(blkid == blk_id_);
+  dec_pending();
+}
+
 void LiteQuery::check_pending() {
   CHECK(pending_ >= 0);
   if (!pending_) {
@@ -627,8 +760,11 @@ bool LiteQuery::make_state_root_proof(Ref<vm::Cell>& proof) {
 bool LiteQuery::make_state_root_proof(Ref<vm::Cell>& proof, Ref<ShardStateQ> state, Ref<BlockData> block,
                                       const BlockIdExt& blkid) {
   CHECK(block.not_null() && state.not_null());
-  auto block_root = block->root_cell();
+  return make_state_root_proof(proof, state->root_cell(), block->root_cell(), blkid);
-  auto state_root = state->root_cell();
+}
+
+bool LiteQuery::make_state_root_proof(Ref<vm::Cell>& proof, Ref<vm::Cell> state_root, Ref<vm::Cell> block_root,
+                                      const BlockIdExt& blkid) {
   CHECK(block_root.not_null() && state_root.not_null());
   RootHash rhash{block_root->get_hash().bits()};
   CHECK(rhash == blkid.root_hash);
@@ -714,6 +850,24 @@ bool LiteQuery::make_shard_info_proof(Ref<vm::Cell>& proof, BlockIdExt& blkid, A
   return true;
 }
 
+bool LiteQuery::make_ancestor_block_proof(Ref<vm::Cell>& proof, Ref<vm::Cell> state_root, const BlockIdExt& old_blkid) {
+  vm::MerkleProofBuilder mpb{std::move(state_root)};
+  auto rconfig = block::ConfigInfo::extract_config(mpb.root(), block::ConfigInfo::needPrevBlocks);
+  if (rconfig.is_error()) {
+    return fatal_error(
+        "cannot extract previous block configuration from masterchain state while constructing Merkle proof for "s +
+        old_blkid.to_str());
+  }
+  if (!rconfig.move_as_ok()->check_old_mc_block_id(old_blkid, true)) {
+    return fatal_error("cannot check that "s + old_blkid.to_str() +
+                       " is indeed a previous masterchain block while constructing Merkle proof");
+  }
+  if (!mpb.extract_proof_to(proof)) {
+    return fatal_error("error while constructing Merkle proof for old masterchain block "s + old_blkid.to_str());
+  }
+  return true;
+}
+
 void LiteQuery::continue_getAccountState() {
   LOG(INFO) << "continue getAccountState() query";
   if (acc_workchain_ == masterchainId) {
@@ -1000,10 +1154,8 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector<int> param_list)
   if (!make_mc_state_root_proof(proof1)) {
     return;
   }
-  auto state_root = mc_state_->root_cell();
+  vm::MerkleProofBuilder mpb{mc_state_->root_cell()};
-  auto usage_tree = std::make_shared<vm::CellUsageTree>();
+  auto res = block::Config::extract_from_state(mpb.root(), mode);
-  auto usage_cell = vm::UsageCell::create(state_root, usage_tree->root_ptr());
-  auto res = block::Config::extract_from_state(usage_cell, mode);
   if (res.is_error()) {
     fatal_error(res.move_as_error());
     return;
@@ -1025,15 +1177,12 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector<int> param_list)
     fatal_error("error while traversing required configuration parameters: "s + err.get_msg());
     return;
   }
-  auto proof2 = vm::MerkleProof::generate(state_root, usage_tree.get());
-  usage_tree.reset();
-  usage_cell.clear();
   auto res1 = vm::std_boc_serialize(std::move(proof1));
   if (res1.is_error()) {
     fatal_error("cannot serialize Merkle proof : "s + res1.move_as_error().to_string());
     return;
   }
-  auto res2 = vm::std_boc_serialize(std::move(proof2));
+  auto res2 = mpb.extract_proof_boc();
   if (res2.is_error()) {
     fatal_error("cannot serialize Merkle proof : "s + res2.move_as_error().to_string());
     return;
@@ -1094,22 +1243,21 @@ void LiteQuery::continue_getShardInfo(ShardIdFull shard, bool exact) {
 
 void LiteQuery::continue_getAllShardsInfo() {
   LOG(INFO) << "completing getAllShardsInfo() query";
-  Ref<vm::Cell> proof1;
+  Ref<vm::Cell> proof1, proof2;
   if (!make_mc_state_root_proof(proof1)) {
     return;
   }
-  auto state_root = mc_state_->root_cell();
+  vm::MerkleProofBuilder mpb{mc_state_->root_cell()};
-  auto usage_tree = std::make_shared<vm::CellUsageTree>();
+  auto shards_dict = block::ShardConfig::extract_shard_hashes_dict(mpb.root());
-  auto usage_cell = vm::UsageCell::create(state_root, usage_tree->root_ptr());
-  auto shards_dict = block::ShardConfig::extract_shard_hashes_dict(usage_cell);
   if (!shards_dict) {
     fatal_error("cannot extract ShardHashes from last mc state");
     return;
   }
-  auto proof2 = vm::MerkleProof::generate(state_root, usage_tree.get());
+  if (!mpb.extract_proof_to(proof2)) {
-  usage_tree.reset();
+    fatal_error("cannot construct Merkle proof for all shards dictionary");
-  usage_cell.clear();
+    return;
-  shards_dict = block::ShardConfig::extract_shard_hashes_dict(state_root);
+  }
+  shards_dict = block::ShardConfig::extract_shard_hashes_dict(mc_state_->root_cell());
   vm::CellBuilder cb;
   Ref<vm::Cell> cell;
   if (!(std::move(shards_dict)->append_dict_to_bool(cb) && cb.finalize_to(cell))) {
@@ -1329,7 +1477,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to,
 
 void LiteQuery::continue_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, int mode,
                                        Ref<MasterchainStateQ> state) {
-  if (mode & 1) {
+  if (!(mode & 1)) {
     base_blk_id_ = to;
     if (!to.is_masterchain_ext()) {
       fatal_error("last masterchain block id "s + to.to_str() + " is invalid");
@@ -1362,7 +1510,7 @@ void LiteQuery::continue_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to,
                 base_blk_id_.to_str());
     return;
   }
-  chain_ = std::make_unique<block::BlkProofChain>(from, to, mode);
+  chain_ = std::make_unique<block::BlockProofChain>(from, to, mode);
   blk_id_ = from;
   construct_proof_chain(from);
 }
@@ -1370,7 +1518,12 @@ void LiteQuery::continue_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to,
 bool LiteQuery::construct_proof_chain(ton::BlockIdExt id) {
   CHECK(chain_);
   if (chain_->link_count() >= 16 || id == chain_->to) {
-    return finish_proof_chain(std::move(id));
+    if (!(chain_->last_link_incomplete() && chain_->last_link().to.seqno())) {
+      return finish_proof_chain(std::move(id));
+    } else {
+      set_continuation([this, id]() { finish_proof_chain(id); });
+      return request_proof_link(id);
+    }
   }
   if (chain_->to.seqno() == id.seqno()) {
     return fatal_error("cannot have two different masterchain blocks "s + chain_->to.to_str() + " and " + id.to_str() +
@@ -1400,6 +1553,28 @@ bool LiteQuery::construct_proof_chain(ton::BlockIdExt id) {
   }
 }
 
+// adjust dest_proof and is_key of the last link of existing proof
+bool LiteQuery::adjust_last_proof_link(ton::BlockIdExt cur, Ref<vm::Cell> block_root) {
+  CHECK(chain_);
+  if (!(chain_->last_link_incomplete() && chain_->last_link().to.seqno())) {
+    return true;
+  }
+  auto& link = chain_->last_link();
+  CHECK(link.dest_proof.is_null());
+  CHECK(link.to == cur);
+  if (cur.root_hash != block_root->get_hash().bits()) {
+    return fatal_error("root hash mismatch in block root of "s + cur.to_str());
+  }
+  vm::MerkleProofBuilder mpb{std::move(block_root)};
+  block::gen::Block::Record blk;
+  block::gen::BlockInfo::Record info;
+  if (!(tlb::unpack_cell(mpb.root(), blk) && tlb::unpack_cell(blk.info, info))) {
+    return fatal_error("cannot unpack header of block "s + cur.to_str());
+  }
+  link.is_key = info.key_block;
+  return mpb.extract_proof_to(link.dest_proof);
+}
+
 bool LiteQuery::construct_proof_link_forward(ton::BlockIdExt cur, ton::BlockIdExt next) {
   LOG(INFO) << "constructing a forward proof link from " << cur.to_str() << " to " << next.to_str();
   if (!(cur.is_masterchain_ext() && next.is_masterchain_ext() && mc_state0_->check_old_mc_block_id(cur) &&
@@ -1410,17 +1585,144 @@ bool LiteQuery::construct_proof_link_forward(ton::BlockIdExt cur, ton::BlockIdEx
   if (cur.seqno() >= next.seqno()) {
     return fatal_error("cannot construct forward proof link from "s + cur.to_str() + " to " + next.to_str());
   }
-  set_continuation([this, cur, next]() { construct_proof_link_back_cont(cur, next); });
+  set_continuation([this, cur, next]() { construct_proof_link_forward_cont(cur, next); });
-  return request_mc_block_data(cur);
+  return (cur.seqno() ? request_proof_link(cur) : request_zero_state(cur)) && request_mc_proof(next);
 }
 
 bool LiteQuery::construct_proof_link_forward_cont(ton::BlockIdExt cur, ton::BlockIdExt next) {
-  // ...
+  LOG(INFO) << "continue constructing a forward proof link from " << cur.to_str() << " to " << next.to_str();
+  CHECK(cur.seqno() ? proof_link_.not_null() && proof_link_->block_id() == cur : !buffer_.empty());
+  CHECK(mc_proof_.not_null() && mc_proof_->block_id() == next);
+  try {
+    Ref<vm::Cell> cur_root, next_root;
+    // virtualize roots
+    ton::validator::ProofQ::VirtualizedProof virt1;
+    if (cur.seqno()) {
+      auto vres1 = proof_link_->get_virtual_root();
+      if (vres1.is_error()) {
+        return fatal_error(vres1.move_as_error());
+      }
+      virt1 = vres1.move_as_ok();
+      cur_root = virt1.root;
+    } else {
+      // for zero state, lazily deserialize buffer_ instead
+      vm::StaticBagOfCellsDbLazy::Options options;
+      options.check_crc32c = true;
+      auto res = vm::StaticBagOfCellsDbLazy::create(vm::BufferSliceBlobView::create(std::move(buffer_)), options);
+      if (res.is_error()) {
+        return fatal_error(res.move_as_error());
+      }
+      virt1.boc = res.move_as_ok();
+      auto t_root = virt1.boc->get_root_cell(0);
+      if (t_root.is_error()) {
+        return fatal_error(t_root.move_as_error());
+      }
+      cur_root = t_root.move_as_ok();
+    }
+    auto vres2 = mc_proof_->get_virtual_root();
+    if (vres2.is_error()) {
+      return fatal_error(vres2.move_as_error());
+    }
+    next_root = vres2.ok().root;
+    if (cur.root_hash != cur_root->get_hash().bits()) {
+      return fatal_error("incorrect root hash in ProofLink for block "s + cur.to_str());
+    }
+    if (next.root_hash != next_root->get_hash().bits()) {
+      return fatal_error("incorrect root hash in ProofLink for block "s + cur.to_str());
+    }
+    // adjust dest_proof and is_key of the last link of existing proof
+    if (!adjust_last_proof_link(cur, cur_root)) {
+      return false;
+    }
+    // extract configuration from current block
+    vm::MerkleProofBuilder cur_mpb{cur_root}, next_mpb{next_root};
+    if (cur.seqno()) {
+      auto err = block::check_block_header(cur_mpb.root(), cur);
+      if (err.is_error()) {
+        return fatal_error("incorrect header in ProofLink for block "s + cur.to_str());
+      }
+    }
+    auto cfg_res = cur.seqno()
+                       ? block::Config::extract_from_key_block(cur_mpb.root(), block::ConfigInfo::needValidatorSet)
+                       : block::Config::extract_from_state(cur_mpb.root(), block::ConfigInfo::needValidatorSet);
+    if (cfg_res.is_error()) {
+      return fatal_error(cfg_res.move_as_error());
+    }
+    auto config = cfg_res.move_as_ok();
+    // unpack header of next block
+    auto err = block::check_block_header(next_mpb.root(), next);
+    if (err.is_error()) {
+      return fatal_error("incorrect header in ProofLink for block "s + next.to_str());
+    }
+    block::gen::Block::Record blk;
+    block::gen::BlockInfo::Record info;
+    if (!(tlb::unpack_cell(next_mpb.root(), blk) && tlb::unpack_cell(blk.info, info))) {
+      return fatal_error("cannot unpack header of block "s + cur.to_str());
+    }
+    // compute validator set
+    ShardIdFull shard{masterchainId};
+    auto nodes = config->compute_validator_set(shard, info.gen_utime, info.gen_catchain_seqno);
+    if (nodes.empty()) {
+      return fatal_error(PSTRING() << "cannot compute validator set for block " << next.to_str() << " with utime "
+                                   << info.gen_utime << " and cc_seqno " << info.gen_catchain_seqno
+                                   << " starting from previous key block " << cur.to_str());
+    }
+    auto vset = Ref<ValidatorSetQ>{true, info.gen_catchain_seqno, shard, std::move(nodes)};
+    if (vset.is_null()) {
+      return fatal_error(PSTRING() << "cannot create validator set for block " << next.to_str() << " with utime "
+                                   << info.gen_utime << " and cc_seqno " << info.gen_catchain_seqno
+                                   << " starting from previous key block " << cur.to_str());
+    }
+    auto vset_hash = vset->get_validator_set_hash();
+    if (vset_hash != info.gen_validator_list_hash_short) {
+      return fatal_error(PSTRING() << "computed validator set for block " << next.to_str() << " with utime "
+                                   << info.gen_utime << " and cc_seqno " << info.gen_catchain_seqno
+                                   << " starting from previous key block " << cur.to_str() << " has hash " << vset_hash
+                                   << " different from " << info.gen_validator_list_hash_short
+                                   << " stated in block header");
+    }
+    // extract signatures
+    auto sig_outer_root = vres2.ok().sig_root;
+    block::gen::BlockSignatures::Record sign_rec;
+    block::gen::BlockSignaturesPure::Record sign_pure;
+    if (!(sig_outer_root.not_null() && tlb::unpack_cell(sig_outer_root, sign_rec) &&
+          tlb::csr_unpack(sign_rec.pure_signatures, sign_pure))) {
+      return fatal_error("cannot extract signature set from proof for block "s + next.to_str());
+    }
+    auto sigs = BlockSignatureSetQ::fetch(sign_pure.signatures->prefetch_ref());
+    if (sigs.is_null()) {
+      return fatal_error("cannot deserialize signature set from proof for block "s + next.to_str());
+    }
+    // check signatures (sanity check; comment later for better performance)
+    /*
+    auto S = vset->check_signatures(next.root_hash, next.file_hash, sigs);
+    if (S.is_error()) {
+      return fatal_error("error checking signatures from proof for block "s + next.to_str() + " : " +
+                         S.move_as_error().to_string());
+    }
+    */
+    // serialize signatures
+    auto& link = chain_->new_link(cur, next, info.key_block);
+    link.cc_seqno = info.gen_catchain_seqno;
+    link.validator_set_hash = info.gen_validator_list_hash_short;
+    link.signatures = std::move(sigs.write().signatures());
+    // serialize proofs
+    if (!(cur_mpb.extract_proof_to(link.proof) && next_mpb.extract_proof_to(link.dest_proof))) {
+      return fatal_error("error constructing Merkle proof for forward proof link from "s + cur.to_str() + " to " +
+                         next.to_str());
+    }
+    // continue constructing from `next`
+    return construct_proof_chain(next);
+  } catch (vm::VmVirtError&) {
+    return fatal_error("virtualization error during construction of forward proof link from "s + cur.to_str() + " to " +
+                       next.to_str());
+  }
   return fatal_error("construction of forward proof links not implemented yet");
 }
 
 bool LiteQuery::construct_proof_link_back(ton::BlockIdExt cur, ton::BlockIdExt next) {
   LOG(INFO) << "constructing a backward proof link from " << cur.to_str() << " to " << next.to_str();
+  CHECK(chain_);
   if (!(cur.is_masterchain_ext() && next.is_masterchain_ext() && mc_state0_->check_old_mc_block_id(cur) &&
         mc_state0_->check_old_mc_block_id(next))) {
     return fatal_error("cannot construct backward proof link from "s + cur.to_str() + " to " + next.to_str() +
@@ -1430,26 +1732,114 @@ bool LiteQuery::construct_proof_link_back(ton::BlockIdExt cur, ton::BlockIdExt n
     return fatal_error("cannot construct backward proof link from "s + cur.to_str() + " to " + next.to_str());
   }
   set_continuation([this, cur, next]() { construct_proof_link_back_cont(cur, next); });
-  return request_mc_block_data_state(cur);
+  return request_proof_link(cur) && request_mc_block_state(cur);
 }
 
 bool LiteQuery::construct_proof_link_back_cont(ton::BlockIdExt cur, ton::BlockIdExt next) {
   LOG(INFO) << "continue constructing a backward proof link from " << cur.to_str() << " to " << next.to_str();
-  CHECK(mc_state_.not_null() && mc_block_.not_null() && mc_state_->get_block_id() == cur &&
+  CHECK(mc_state_.not_null() && proof_link_.not_null() && mc_state_->get_block_id() == cur &&
-        mc_block_->block_id() == cur);
+        proof_link_->block_id() == cur);
-  // ...
+  try {
-  return fatal_error("construction of backward proof links not implemented yet");
+    // virtualize proof link
+    auto vres1 = proof_link_->get_virtual_root();
+    if (vres1.is_error()) {
+      return fatal_error(vres1.move_as_error());
+    }
+    auto vroot = vres1.ok().root;
+    // adjust dest_proof and is_key of the last link of existing proof
+    if (!adjust_last_proof_link(cur, vroot)) {
+      return false;
+    }
+    // construct proof that `mc_state_` is the state of `cur`
+    Ref<vm::Cell> state_proof, proof;
+    if (!make_state_root_proof(proof, mc_state_->root_cell(), vroot, cur)) {
+      return fatal_error("cannot construct proof for state of masterchain block "s + cur.to_str());
+    }
+    // construct proof that `next` is listed in OldMcBlocksInfo of `mc_state_`
+    if (!make_ancestor_block_proof(state_proof, mc_state_->root_cell(), next)) {
+      return fatal_error("cannot prove that "s + next.to_str() +
+                         " is in the previous block set of the masterchain state of " + cur.to_str());
+    }
+    // create a BlockProofLink for cur -> next (without dest_proof)
+    auto& link = chain_->new_link(cur, next, !next.seqno());
+    link.proof = std::move(proof);
+    link.state_proof = std::move(state_proof);
+    // continue constructing proof chain from `next`
+    return construct_proof_chain(next);
+  } catch (vm::VmVirtError&) {
+    return fatal_error("virtualization error during construction of backward proof link from "s + cur.to_str() +
+                       " to " + next.to_str());
+  }
 }
 
 bool LiteQuery::finish_proof_chain(ton::BlockIdExt id) {
   CHECK(chain_);
   LOG(INFO) << "finish constructing block proof chain from " << chain_->from.to_str() << " to " << chain_->to.to_str()
             << " (constructed " << chain_->link_count() << " up to " << id.to_str() << ")";
-  // bool complete = (id == chain_->to);
+  try {
-  chain_->to = id;
+    if (chain_->last_link_incomplete() && chain_->last_link().to.seqno()) {
-  // serialize answer
+      CHECK(proof_link_.not_null() && proof_link_->block_id() == id);
-  // ...
+      auto vres1 = proof_link_->get_virtual_root();
-  return fatal_error("cannot serialize PartialBlockProof from "s + chain_->from.to_str() + " to " + id.to_str());
+      if (vres1.is_error()) {
+        return fatal_error(vres1.move_as_error());
+      }
+      if (!adjust_last_proof_link(id, vres1.ok().root)) {
+        return false;
+      }
+    }
+    chain_->complete = (id == chain_->to);
+    chain_->to = id;
+    // serialize answer
+    std::vector<ton::tl_object_ptr<lite_api::liteServer_BlockLink>> a;
+    for (auto& link : chain_->links) {
+      td::BufferSlice dest_proof_boc;
+      if (link.to.seqno()) {
+        auto res = vm::std_boc_serialize(link.dest_proof);
+        if (res.is_error()) {
+          return fatal_error("error while serializing destination block Merkle proof in block proof link from "s +
+                             link.from.to_str() + " to " + link.to.to_str() + " : " + res.move_as_error().to_string());
+        }
+        dest_proof_boc = res.move_as_ok();
+      }
+      auto src_proof_boc = vm::std_boc_serialize(link.proof);
+      if (src_proof_boc.is_error()) {
+        return fatal_error("error while serializing source block Merkle proof in block proof link from "s +
+                           link.from.to_str() + " to " + link.to.to_str() + " : " +
+                           src_proof_boc.move_as_error().to_string());
+      }
+      if (link.is_fwd) {
+        // serialize forward link
+        std::vector<ton::tl_object_ptr<lite_api::liteServer_signature>> b;
+        for (auto& sig : link.signatures) {
+          b.push_back(create_tl_object<lite_api::liteServer_signature>(sig.node, std::move(sig.signature)));
+        }
+        a.push_back(create_tl_object<lite_api::liteServer_blockLinkForward>(
+            link.is_key, ton::create_tl_lite_block_id(link.from), ton::create_tl_lite_block_id(link.to),
+            std::move(dest_proof_boc), src_proof_boc.move_as_ok(),
+            create_tl_object<lite_api::liteServer_signatureSet>(link.validator_set_hash, link.cc_seqno, std::move(b))));
+      } else {
+        // serialize backward link
+        auto state_proof_boc = vm::std_boc_serialize(link.state_proof);
+        if (state_proof_boc.is_error()) {
+          return fatal_error("error while serializing source state Merkle proof in block proof link from "s +
+                             link.from.to_str() + " to " + link.to.to_str() + " : " +
+                             state_proof_boc.move_as_error().to_string());
+        }
+        a.push_back(create_tl_object<lite_api::liteServer_blockLinkBack>(
+            link.is_key, ton::create_tl_lite_block_id(link.from), ton::create_tl_lite_block_id(link.to),
+            std::move(dest_proof_boc), src_proof_boc.move_as_ok(), state_proof_boc.move_as_ok()));
+      }
+    }
+    LOG(INFO) << "getBlockProof() query completed";
+    auto c = ton::create_serialize_tl_object<ton::lite_api::liteServer_partialBlockProof>(
+        chain_->complete, ton::create_tl_lite_block_id(chain_->from), ton::create_tl_lite_block_id(chain_->to),
+        std::move(a));
+    return finish_query(std::move(c));
+  } catch (vm::VmError& err) {
+    return fatal_error("vm error while constructing block proof chain : "s + err.get_msg());
+  } catch (vm::VmVirtError& err) {
+    return fatal_error("virtualization error while constructing block proof chain : "s + err.get_msg());
+  }
 }
 
 }  // namespace validator

validator/impl/liteserver.hpp

@@ -24,6 +24,7 @@
 #include "interfaces/validator-manager.h"
 #include "interfaces/shard.h"
 #include "shard.hpp"
+#include "proof.hpp"
 
 namespace ton {
 
@@ -40,20 +41,25 @@ class LiteQuery : public td::actor::Actor {
   StdSmcAddress acc_addr_;
   LogicalTime trans_lt_;
   Bits256 trans_hash_;
-  BlockIdExt base_blk_id_, blk_id_;
+  BlockIdExt base_blk_id_, base_blk_id_alt_, blk_id_;
   Ref<MasterchainStateQ> mc_state_, mc_state0_;
   Ref<ShardStateQ> state_;
   Ref<BlockData> mc_block_, block_;
+  Ref<ProofQ> mc_proof_, mc_proof_alt_;
+  Ref<ProofLinkQ> proof_link_;
+  td::BufferSlice buffer_;
   std::function<void()> continuation_;
   bool cont_set_{false};
   td::BufferSlice shard_proof_;
   std::vector<Ref<vm::Cell>> roots_;
   std::vector<Ref<td::CntObject>> aux_objs_;
   std::vector<ton::BlockIdExt> blk_ids_;
-  std::unique_ptr<block::BlkProofChain> chain_;
+  std::unique_ptr<block::BlockProofChain> chain_;
 
  public:
   static constexpr double default_timeout_seconds = 4.5;
+  static constexpr int ls_version = 0x101;         // 1.1
+  static constexpr long long ls_capabilities = 1;  // +1 = build block proof chains
   LiteQuery(td::BufferSlice data, td::actor::ActorId<ton::validator::ValidatorManager> manager,
             td::Promise<td::BufferSlice> promise);
   static void run_query(td::BufferSlice data, td::actor::ActorId<ton::validator::ValidatorManager> manager,
@@ -64,6 +70,7 @@ class LiteQuery : public td::actor::Actor {
   bool fatal_error(std::string err_msg, int err_code = -400);
   bool fatal_error(int err_code, std::string err_msg = "");
   void abort_query(td::Status reason);
+  void abort_query_ext(td::Status reason, std::string err_msg);
   bool finish_query(td::BufferSlice result);
   void alarm() override;
   void start_up() override;
@@ -77,6 +84,7 @@ class LiteQuery : public td::actor::Actor {
   void continue_getBlockHeader(BlockIdExt blkid, int mode, Ref<BlockData> block);
   void perform_getState(BlockIdExt blkid);
   void continue_getState(BlockIdExt blkid, Ref<ShardState> state);
+  void continue_getZeroState(BlockIdExt blkid, td::BufferSlice state);
   void perform_sendMessage(td::BufferSlice ext_msg);
   void perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr);
   void continue_getAccountState_0(Ref<MasterchainState> mc_state, BlockIdExt blkid);
@@ -105,18 +113,25 @@ class LiteQuery : public td::actor::Actor {
   bool construct_proof_link_forward_cont(ton::BlockIdExt cur, ton::BlockIdExt next);
   bool construct_proof_link_back(ton::BlockIdExt cur, ton::BlockIdExt next);
   bool construct_proof_link_back_cont(ton::BlockIdExt cur, ton::BlockIdExt next);
+  bool adjust_last_proof_link(ton::BlockIdExt cur, Ref<vm::Cell> block_root);
   bool finish_proof_chain(ton::BlockIdExt id);
 
   bool request_block_data(BlockIdExt blkid);
   bool request_block_state(BlockIdExt blkid);
   bool request_block_data_state(BlockIdExt blkid);
+  bool request_proof_link(BlockIdExt blkid);
   bool request_mc_block_data(BlockIdExt blkid);
   bool request_mc_block_state(BlockIdExt blkid);
   bool request_mc_block_data_state(BlockIdExt blkid);
+  bool request_mc_proof(BlockIdExt blkid, int mode = 0);
+  bool request_zero_state(BlockIdExt blkid);
   void got_block_state(BlockIdExt blkid, Ref<ShardState> state);
   void got_mc_block_state(BlockIdExt blkid, Ref<ShardState> state);
   void got_block_data(BlockIdExt blkid, Ref<BlockData> data);
   void got_mc_block_data(BlockIdExt blkid, Ref<BlockData> data);
+  void got_mc_block_proof(BlockIdExt blkid, int mode, Ref<Proof> proof);
+  void got_block_proof_link(BlockIdExt blkid, Ref<ProofLink> proof_link);
+  void got_zero_state(BlockIdExt blkid, td::BufferSlice zerostate);
   void dec_pending() {
     if (!--pending_) {
       check_pending();
@@ -128,11 +143,14 @@ class LiteQuery : public td::actor::Actor {
   bool make_state_root_proof(Ref<vm::Cell>& proof);
   bool make_state_root_proof(Ref<vm::Cell>& proof, Ref<ShardStateQ> state, Ref<BlockData> block,
                              const BlockIdExt& blkid);
+  bool make_state_root_proof(Ref<vm::Cell>& proof, Ref<vm::Cell> state_root, Ref<vm::Cell> block_root,
+                             const BlockIdExt& blkid);
   bool make_shard_info_proof(Ref<vm::Cell>& proof, vm::CellSlice& cs, ShardIdFull shard, ShardIdFull& true_shard,
                              Ref<vm::Cell>& leaf, bool& found, bool exact = true);
   bool make_shard_info_proof(Ref<vm::Cell>& proof, Ref<block::McShardHash>& info, ShardIdFull shard, bool exact = true);
   bool make_shard_info_proof(Ref<vm::Cell>& proof, Ref<block::McShardHash>& info, AccountIdPrefixFull prefix);
   bool make_shard_info_proof(Ref<vm::Cell>& proof, BlockIdExt& blkid, AccountIdPrefixFull prefix);
+  bool make_ancestor_block_proof(Ref<vm::Cell>& proof, Ref<vm::Cell> state_root, const BlockIdExt& old_blkid);
 };
 
 }  // namespace validator

validator/impl/proof.cpp

@@ -47,11 +47,11 @@ td::Result<BlockSeqno> ProofLinkQ::prev_key_mc_seqno() const {
   //  return td::Status::Error(
   //      -668, "cannot compute previous key masterchain block from ProofLink of non-masterchain block "s + id_.to_str());
   //}
-  TRY_RESULT(pair, get_virtual_root(true));
+  TRY_RESULT(virt, get_virtual_root(true));
   try {
     block::gen::Block::Record blk;
     block::gen::BlockInfo::Record info;
-    if (!(tlb::unpack_cell(std::move(pair.first), blk) && tlb::unpack_cell(blk.info, info) && !info.version)) {
+    if (!(tlb::unpack_cell(std::move(virt.root), blk) && tlb::unpack_cell(blk.info, info) && !info.version)) {
       return td::Status::Error(-668,
                                "cannot unpack block header in the Merkle proof for masterchain block "s + id_.to_str());
     }
@@ -66,10 +66,10 @@ td::Result<td::Ref<ConfigHolder>> ProofLinkQ::get_key_block_config() const {
     return td::Status::Error(
         -668, "cannot compute previous key masterchain block from ProofLink of non-masterchain block "s + id_.to_str());
   }
-  TRY_RESULT(pair, get_virtual_root(true));
+  TRY_RESULT(virt, get_virtual_root(true));
   try {
-    TRY_RESULT(cfg, block::Config::extract_from_key_block(std::move(pair.first), block::Config::needValidatorSet));
+    TRY_RESULT(cfg, block::Config::extract_from_key_block(std::move(virt.root), block::Config::needValidatorSet));
-    return td::make_ref<ConfigHolderQ>(std::move(cfg), std::move(pair.second));
+    return td::make_ref<ConfigHolderQ>(std::move(cfg), std::move(virt.boc));
   } catch (vm::VmVirtError &) {
     return td::Status::Error(-668,
                              "virtualization error while traversing masterchain block proof for "s + id_.to_str());
@@ -78,11 +78,11 @@ td::Result<td::Ref<ConfigHolder>> ProofLinkQ::get_key_block_config() const {
 
 td::Result<ProofLink::BasicHeaderInfo> ProofLinkQ::get_basic_header_info() const {
   BasicHeaderInfo res;
-  TRY_RESULT(pair, get_virtual_root(true));
+  TRY_RESULT(virt, get_virtual_root(true));
   try {
     block::gen::Block::Record blk;
     block::gen::BlockInfo::Record info;
-    if (!(tlb::unpack_cell(std::move(pair.first), blk) && tlb::unpack_cell(blk.info, info) && !info.version)) {
+    if (!(tlb::unpack_cell(std::move(virt.root), blk) && tlb::unpack_cell(blk.info, info) && !info.version)) {
       return td::Status::Error(-668,
                                "cannot unpack block header in the Merkle proof for masterchain block "s + id_.to_str());
     }
@@ -96,8 +96,7 @@ td::Result<ProofLink::BasicHeaderInfo> ProofLinkQ::get_basic_header_info() const
   }
 }
 
-td::Result<std::pair<Ref<vm::Cell>, std::shared_ptr<vm::StaticBagOfCellsDb>>> ProofLinkQ::get_virtual_root(
+td::Result<ProofLinkQ::VirtualizedProof> ProofLinkQ::get_virtual_root(bool lazy) const {
-    bool lazy) const {
   if (data_.empty()) {
     return td::Status::Error(-668, "block proof is empty");
   }
@@ -143,7 +142,23 @@ td::Result<std::pair<Ref<vm::Cell>, std::shared_ptr<vm::StaticBagOfCellsDb>>> Pr
                                        " contains a Merkle proof with incorrect root hash: expected " +
                                        proof_blk_id.root_hash.to_hex() + ", found " + virt_hash.to_hex());
   }
-  return std::make_pair(std::move(virt_root), std::move(boc));
+  return VirtualizedProof{std::move(virt_root), proof.signatures->prefetch_ref(), std::move(boc)};
+}
+
+td::Result<Ref<vm::Cell>> ProofQ::get_signatures_root() const {
+  if (data_.empty()) {
+    return td::Status::Error(-668, "block proof is empty");
+  }
+  TRY_RESULT(root, vm::std_boc_deserialize(data_.as_slice()));
+  block::gen::BlockProof::Record proof;
+  BlockIdExt proof_blk_id;
+  if (!(tlb::unpack_cell(root, proof) && block::tlb::t_BlockIdExt.unpack(proof.proof_for.write(), proof_blk_id))) {
+    return td::Status::Error(-668, "masterchain block proof is invalid");
+  }
+  if (proof_blk_id != id_) {
+    return td::Status::Error(-668, "masterchain block proof is for another block");
+  }
+  return proof.signatures->prefetch_ref();
 }
 
 }  // namespace validator

validator/impl/proof.hpp

@@ -49,9 +49,20 @@ class ProofLinkQ : virtual public ProofLink {
   td::Result<td::Ref<ConfigHolder>> get_key_block_config() const override;
   td::Result<BasicHeaderInfo> get_basic_header_info() const override;
 
- protected:
+  struct VirtualizedProof {
-  td::Result<std::pair<Ref<vm::Cell>, std::shared_ptr<vm::StaticBagOfCellsDb>>> get_virtual_root(
+    Ref<vm::Cell> root, sig_root;
-      bool lazy = false) const;
+    std::shared_ptr<vm::StaticBagOfCellsDb> boc;
+    VirtualizedProof() = default;
+    VirtualizedProof(Ref<vm::Cell> _vroot, Ref<vm::Cell> _sigroot, std::shared_ptr<vm::StaticBagOfCellsDb> _boc)
+        : root(std::move(_vroot)), sig_root(std::move(_sigroot)), boc(std::move(_boc)) {
+    }
+    void clear() {
+      root.clear();
+      sig_root.clear();
+      boc.reset();
+    }
+  };
+  td::Result<VirtualizedProof> get_virtual_root(bool lazy = false) const;
 };
 
 #if TD_MSVC
@@ -66,6 +77,7 @@ class ProofQ : public Proof, public ProofLinkQ {
     return new ProofQ(id_, data_.clone());
   }
   td::Result<Ref<ProofLink>> export_as_proof_link() const override;
+  td::Result<Ref<vm::Cell>> get_signatures_root() const;
 };
 #if TD_MSVC
 #pragma warning(pop)

validator/impl/validate-query.cpp

@@ -255,7 +255,7 @@ void ValidateQuery::start_up() {
     LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager";
     ++pending;
     td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(),
-                                  timeout, [ self = get_self(), i ](td::Result<Ref<ShardState>> res)->void {
+                                  timeout, [self = get_self(), i](td::Result<Ref<ShardState>> res) -> void {
                                     LOG(DEBUG) << "got answer to wait_block_state_short query #" << i;
                                     td::actor::send_closure_later(
                                         std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res));
@@ -269,16 +269,16 @@ void ValidateQuery::start_up() {
   // 5. request masterchain state referred to in the block
   if (!is_masterchain()) {
     ++pending;
-    td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(),
+    td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(), timeout,
-                                  timeout, [self = get_self()](td::Result<Ref<ShardState>> res) {
+                                  [self = get_self()](td::Result<Ref<ShardState>> res) {
                                     LOG(DEBUG) << "got answer to wait_block_state() query for masterchain block";
                                     td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state,
                                                                   std::move(res));
                                   });
     // 5.1. request corresponding block handle
     ++pending;
-    td::actor::send_closure_later(manager, &ValidatorManager::get_block_handle, mc_blkid_,
+    td::actor::send_closure_later(manager, &ValidatorManager::get_block_handle, mc_blkid_, true,
-                                  true, [self = get_self()](td::Result<BlockHandle> res) {
+                                  [self = get_self()](td::Result<BlockHandle> res) {
                                     LOG(DEBUG) << "got answer to get_block_handle() query for masterchain block";
                                     td::actor::send_closure_later(std::move(self), &ValidateQuery::got_mc_handle,
                                                                   std::move(res));
@@ -1008,7 +1008,8 @@ bool ValidateQuery::compute_next_state() {
     return reject_query("header of new state claims it belongs to block "s + hdr_id.to_str() + " instead of " +
                         id_.id.to_str());
   }
-  if (info.custom->size_refs() != is_masterchain()) {
+  CHECK(info.custom->size_refs() == 0 || info.custom->size_refs() == 1);
+  if (info.custom->size_refs() != static_cast<unsigned>(is_masterchain())) {
     return reject_query("McStateExtra in the new state of a non-masterchain block, or conversely");
   }
   if (is_masterchain()) {
@@ -1166,7 +1167,7 @@ bool ValidateQuery::request_neighbor_queues() {
     LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str();
     ++pending;
     send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout,
-                       [ self = get_self(), i ](td::Result<Ref<MessageQueue>> res) {
+                       [self = get_self(), i](td::Result<Ref<MessageQueue>> res) {
                          td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i,
                                                  std::move(res));
                        });
@@ -1284,12 +1285,13 @@ bool ValidateQuery::request_aux_mc_state(BlockSeqno seqno, Ref<MasterchainStateQ
   CHECK(blkid.is_valid_ext() && blkid.is_masterchain());
   LOG(DEBUG) << "sending auxiliary wait_block_state() query for " << blkid.to_str() << " to Manager";
   ++pending;
-  td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, [
+  td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout,
-    self = get_self(), blkid
+                                [self = get_self(), blkid](td::Result<Ref<ShardState>> res) {
-  ](td::Result<Ref<ShardState>> res) {
+                                  LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str();
-    LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str();
+                                  td::actor::send_closure_later(std::move(self),
-    td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_aux_shard_state, blkid, std::move(res));
+                                                                &ValidateQuery::after_get_aux_shard_state, blkid,
-  });
+                                                                std::move(res));
+                                });
   state.clear();
   return true;
 }
@@ -1625,8 +1627,8 @@ bool ValidateQuery::check_shard_layout() {
   WorkchainId wc_id{ton::workchainInvalid};
   Ref<block::WorkchainInfo> wc_info;
 
-  if (!new_shard_conf_->process_sibling_shard_hashes([ self = this, &wc_set, &wc_id, &wc_info, &ccvc ](
+  if (!new_shard_conf_->process_sibling_shard_hashes([self = this, &wc_set, &wc_id, &wc_info, &ccvc](
-          block::McShardHash & cur, const block::McShardHash* sibling) {
+                                                         block::McShardHash& cur, const block::McShardHash* sibling) {
         if (!cur.is_valid()) {
           return -2;
         }
@@ -4617,8 +4619,8 @@ bool ValidateQuery::check_one_library_update(td::ConstBitPtr key, Ref<vm::CellSl
     old_publishers = std::make_unique<vm::Dictionary>(256);
   }
   if (!old_publishers->scan_diff(*new_publishers,
-                                 [ this, lib_key = key ](td::ConstBitPtr key, int key_len, Ref<vm::CellSlice> old_val,
+                                 [this, lib_key = key](td::ConstBitPtr key, int key_len, Ref<vm::CellSlice> old_val,
-                                                         Ref<vm::CellSlice> new_val) {
+                                                       Ref<vm::CellSlice> new_val) {
                                    CHECK(key_len == 256);
                                    if (old_val.not_null() && !old_val->empty_ext()) {
                                      return false;

validator/manager-disk.cpp

@@ -822,7 +822,7 @@ void ValidatorManagerImpl::send_top_shard_block_description(td::Ref<ShardTopBloc
 }
 
 void ValidatorManagerImpl::start_up() {
-  db_ = create_db_actor(actor_id(this), db_root_);
+  db_ = create_db_actor(actor_id(this), db_root_, opts_->get_filedb_depth());
 
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<ValidatorManagerInitResult> R) {
     R.ensure();

validator/manager.cpp

@@ -1275,7 +1275,7 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast) {
 }
 
 void ValidatorManagerImpl::start_up() {
-  db_ = create_db_actor(actor_id(this), db_root_);
+  db_ = create_db_actor(actor_id(this), db_root_, opts_->get_filedb_depth());
   token_manager_ = td::actor::create_actor<TokenManager>("tokenmanager");
 
   auto Q =
@@ -1704,6 +1704,33 @@ void ValidatorManagerImpl::allow_archive(BlockIdExt block_id, td::Promise<bool>
   td::actor::send_closure(db_, &Db::archive, block_id, std::move(P));
 }
 
+void ValidatorManagerImpl::allow_delete(BlockIdExt block_id, td::Promise<bool> promise) {
+  auto key_ttl = td::Clocks::system() - opts_->key_proof_ttl();
+  auto ttl = td::Clocks::system() - opts_->archive_ttl();
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), promise = std::move(promise), ttl, key_ttl](td::Result<BlockHandle> R) mutable {
+        if (R.is_error()) {
+          promise.set_result(true);
+          return;
+        }
+        auto handle = R.move_as_ok();
+        if (!handle->moved_to_storage()) {
+          promise.set_result(false);
+          return;
+        }
+        if (!handle->inited_unix_time()) {
+          promise.set_result(true);
+          return;
+        }
+        if (!handle->inited_is_key_block() || !handle->is_key_block()) {
+          promise.set_result(handle->unix_time() <= ttl);
+        } else {
+          promise.set_result(handle->unix_time() <= key_ttl);
+        }
+      });
+  get_block_handle(block_id, false, std::move(P));
+}
+
 void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise<bool> promise) {
   if (!gc_masterchain_handle_) {
     promise.set_result(false);

validator/manager.hpp

@@ -455,12 +455,13 @@ class ValidatorManagerImpl : public ValidatorManager {
   }
 
  public:
+  void allow_delete(BlockIdExt block_id, td::Promise<bool> promise);
   void allow_archive(BlockIdExt block_id, td::Promise<bool> promise);
   void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise<bool> promise) override {
     if (!is_archive) {
       allow_archive(block_id, std::move(promise));
     } else {
-      promise.set_result(false);
+      allow_delete(block_id, std::move(promise));
     }
   }
   void allow_block_state_gc(BlockIdExt block_id, td::Promise<bool> promise) override;
@@ -476,21 +477,21 @@ class ValidatorManagerImpl : public ValidatorManager {
     if (!is_archive) {
       allow_archive(block_id, std::move(promise));
     } else {
-      promise.set_result(false);
+      allow_delete(block_id, std::move(promise));
     }
   }
   void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise<bool> promise) override {
     if (!is_archive) {
       allow_archive(block_id, std::move(promise));
     } else {
-      promise.set_result(false);
+      allow_delete(block_id, std::move(promise));
     }
   }
   void allow_block_candidate_gc(BlockIdExt block_id, td::Promise<bool> promise) override {
     allow_block_state_gc(block_id, std::move(promise));
   }
   void allow_block_info_gc(BlockIdExt block_id, td::Promise<bool> promise) override {
-    promise.set_result(false);
+    allow_delete(block_id, std::move(promise));
   }
 
   void send_peek_key_block_request();

validator/validator-group.cpp

@@ -87,6 +87,10 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s
                                        sig_set, approve_sig_set,
                                        promise = std::move(promise)](td::Result<td::Unit> R) mutable {
     if (R.is_error()) {
+      if (R.error().code() == ErrorCode::cancelled) {
+        promise.set_value(td::Unit());
+        return;
+      }
       LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error();
       td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block),
                               std::move(prev), std::move(sig_set), std::move(approve_sig_set), std::move(promise));

validator/validator-options.cpp

@@ -27,10 +27,11 @@ namespace validator {
 td::Ref<ValidatorManagerOptions> ValidatorManagerOptions::create(
     BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function<bool(ShardIdFull)> check_shard,
     bool allow_blockchain_init, td::ClocksBase::Duration sync_blocks_before, td::ClocksBase::Duration block_ttl,
-    td::ClocksBase::Duration state_ttl, bool initial_sync_disabled) {
+    td::ClocksBase::Duration state_ttl, td::ClocksBase::Duration archive_ttl, td::ClocksBase::Duration key_proof_ttl,
+    bool initial_sync_disabled) {
   return td::make_ref<ValidatorManagerOptionsImpl>(zero_block_id, init_block_id, std::move(check_shard),
                                                    allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl,
-                                                   initial_sync_disabled);
+                                                   archive_ttl, key_proof_ttl, initial_sync_disabled);
 }
 
 }  // namespace validator

validator/validator-options.hpp

@@ -47,6 +47,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   td::ClocksBase::Duration state_ttl() const override {
     return state_ttl_;
   }
+  td::ClocksBase::Duration archive_ttl() const override {
+    return archive_ttl_;
+  }
+  td::ClocksBase::Duration key_proof_ttl() const override {
+    return key_proof_ttl_;
+  }
   bool initial_sync_disabled() const override {
     return initial_sync_disabled_;
   }
@@ -70,6 +76,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
     }
     return static_cast<td::uint32>(best);
   }
+  td::uint32 get_filedb_depth() const override {
+    return db_depth_;
+  }
 
   void set_zero_block_id(BlockIdExt block_id) override {
     zero_block_id_ = block_id;
@@ -92,12 +101,22 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   void set_state_ttl(td::ClocksBase::Duration value) override {
     state_ttl_ = value;
   }
+  void set_archive_ttl(td::ClocksBase::Duration value) override {
+    archive_ttl_ = value;
+  }
+  void set_key_proof_ttl(td::ClocksBase::Duration value) override {
+    key_proof_ttl_ = value;
+  }
   void set_initial_sync_disabled(bool value) override {
     initial_sync_disabled_ = value;
   }
   void set_hardforks(std::vector<BlockIdExt> vec) override {
     hardforks_ = std::move(vec);
   }
+  void set_filedb_depth(td::uint32 value) override {
+    CHECK(value <= 32);
+    db_depth_ = value;
+  }
 
   ValidatorManagerOptionsImpl *make_copy() const override {
     return new ValidatorManagerOptionsImpl(*this);
@@ -106,7 +125,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id,
                               std::function<bool(ShardIdFull)> check_shard, bool allow_blockchain_init,
                               td::ClocksBase::Duration sync_blocks_before, td::ClocksBase::Duration block_ttl,
-                              td::ClocksBase::Duration state_ttl, bool initial_sync_disabled)
+                              td::ClocksBase::Duration archive_ttl, td::ClocksBase::Duration state_ttl,
+                              td::ClocksBase::Duration key_proof_ttl, bool initial_sync_disabled)
       : zero_block_id_(zero_block_id)
       , init_block_id_(init_block_id)
       , check_shard_(std::move(check_shard))
@@ -114,6 +134,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
       , sync_blocks_before_(sync_blocks_before)
       , block_ttl_(block_ttl)
       , state_ttl_(state_ttl)
+      , archive_ttl_(archive_ttl)
+      , key_proof_ttl_(key_proof_ttl)
       , initial_sync_disabled_(initial_sync_disabled) {
   }
 
@@ -125,8 +147,11 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   td::ClocksBase::Duration sync_blocks_before_;
   td::ClocksBase::Duration block_ttl_;
   td::ClocksBase::Duration state_ttl_;
+  td::ClocksBase::Duration archive_ttl_;
+  td::ClocksBase::Duration key_proof_ttl_;
   bool initial_sync_disabled_;
   std::vector<BlockIdExt> hardforks_;
+  td::uint32 db_depth_ = 2;
 };
 
 }  // namespace validator

validator/validator.h

@@ -53,9 +53,12 @@ struct ValidatorManagerOptions : public td::CntObject {
   virtual td::ClocksBase::Duration sync_blocks_before() const = 0;
   virtual td::ClocksBase::Duration block_ttl() const = 0;
   virtual td::ClocksBase::Duration state_ttl() const = 0;
+  virtual td::ClocksBase::Duration archive_ttl() const = 0;
+  virtual td::ClocksBase::Duration key_proof_ttl() const = 0;
   virtual bool initial_sync_disabled() const = 0;
   virtual bool is_hardfork(BlockIdExt block_id) const = 0;
   virtual td::uint32 get_vertical_height(BlockSeqno seqno) const = 0;
+  virtual td::uint32 get_filedb_depth() const = 0;
 
   virtual void set_zero_block_id(BlockIdExt block_id) = 0;
   virtual void set_init_block_id(BlockIdExt block_id) = 0;
@@ -64,14 +67,18 @@ struct ValidatorManagerOptions : public td::CntObject {
   virtual void set_sync_blocks_before(td::ClocksBase::Duration value) = 0;
   virtual void set_block_ttl(td::ClocksBase::Duration value) = 0;
   virtual void set_state_ttl(td::ClocksBase::Duration value) = 0;
+  virtual void set_archive_ttl(td::ClocksBase::Duration value) = 0;
+  virtual void set_key_proof_ttl(td::ClocksBase::Duration value) = 0;
   virtual void set_initial_sync_disabled(bool value) = 0;
   virtual void set_hardforks(std::vector<BlockIdExt> hardforks) = 0;
+  virtual void set_filedb_depth(td::uint32 value) = 0;
 
   static td::Ref<ValidatorManagerOptions> create(
       BlockIdExt zero_block_id, BlockIdExt init_block_id,
       std::function<bool(ShardIdFull)> check_shard = [](ShardIdFull) { return true; },
       bool allow_blockchain_init = false, td::ClocksBase::Duration sync_blocks_before = 300,
       td::ClocksBase::Duration block_ttl = 86400 * 7, td::ClocksBase::Duration state_ttl = 3600,
+      td::ClocksBase::Duration archive_ttl = 86400 * 365, td::ClocksBase::Duration key_proof_ttl = 86400 * 3650,
       bool initial_sync_disabled = false);
 };