Accelerator, part 1 (#1119)

This commit contains some parts of https://github.com/ton-blockchain/ton/tree/accelerator
This is auxiliary code that mostly does not change node behavior.

1) Semiprivate overlays and other improvements in overlays code
2) Rename actual_min_split -> monitor_min_split, fix building shard overlays
3) Loading block candidates by block id from DB, fix accept_block after validator restart
4) Cells: ProofStorageStat and changes in CellUsageTree
5) Remove some unused code, other minor changes

crypto/block/block.tlb

@@ -666,15 +666,15 @@ wc_split_merge_timings#0
 //workchain#a5 enabled_since:uint32 min_split:(## 8) max_split:(## 8)
 //  { min_split <= max_split } { max_split <= 60 }
 
-workchain#a6 enabled_since:uint32 actual_min_split:(## 8) 
-  min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split }
+workchain#a6 enabled_since:uint32 monitor_min_split:(## 8)
+  min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split }
   basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 }
   zerostate_root_hash:bits256 zerostate_file_hash:bits256
   version:uint32 format:(WorkchainFormat basic)
   = WorkchainDescr;
 
-workchain_v2#a7 enabled_since:uint32 actual_min_split:(## 8)
-  min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split }
+workchain_v2#a7 enabled_since:uint32 monitor_min_split:(## 8)
+  min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split }
   basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 }
   zerostate_root_hash:bits256 zerostate_file_hash:bits256
   version:uint32 format:(WorkchainFormat basic)

crypto/block/mc-config.cpp

@@ -2075,7 +2075,7 @@ bool WorkchainInfo::unpack(ton::WorkchainId wc, vm::CellSlice& cs) {
   }
   auto unpack_v1 = [this](auto& info) {
     enabled_since = info.enabled_since;
-    actual_min_split = info.actual_min_split;
+    monitor_min_split = info.monitor_min_split;
     min_split = info.min_split;
     max_split = info.max_split;
     basic = info.basic;

crypto/block/mc-config.h

@@ -414,7 +414,7 @@ struct CatchainValidatorsConfig {
 struct WorkchainInfo : public td::CntObject {
   ton::WorkchainId workchain{ton::workchainInvalid};
   ton::UnixTime enabled_since;
-  td::uint32 actual_min_split;
+  td::uint32 monitor_min_split;
   td::uint32 min_split, max_split;
   bool basic;
   bool active;

crypto/vm/boc.cpp

@@ -1214,4 +1214,35 @@ bool VmStorageStat::add_storage(const CellSlice& cs) {
   return true;
 }
 
+static td::uint64 estimate_prunned_size() {
+  return 41;
+}
+
+static td::uint64 estimate_serialized_size(const Ref<DataCell>& cell) {
+  return cell->get_serialized_size() + cell->size_refs() * 3 + 3;
+}
+
+void ProofStorageStat::add_cell(const Ref<DataCell>& cell) {
+  auto& status = cells_[cell->get_hash()];
+  if (status == c_loaded) {
+    return;
+  }
+  if (status == c_prunned) {
+    proof_size_ -= estimate_prunned_size();
+  }
+  status = c_loaded;
+  proof_size_ += estimate_serialized_size(cell);
+  for (unsigned i = 0; i < cell->size_refs(); ++i) {
+    auto& child_status = cells_[cell->get_ref(i)->get_hash()];
+    if (child_status == c_none) {
+      child_status = c_prunned;
+      proof_size_ += estimate_prunned_size();
+    }
+  }
+}
+
+td::uint64 ProofStorageStat::estimate_proof_size() const {
+  return proof_size_;
+}
+
 }  // namespace vm

crypto/vm/boc.h

@@ -163,6 +163,18 @@ struct VmStorageStat {
   }
 };
 
+class ProofStorageStat {
+ public:
+  void add_cell(const Ref<DataCell>& cell);
+  td::uint64 estimate_proof_size() const;
+ private:
+  enum CellStatus {
+    c_none = 0, c_prunned = 1, c_loaded = 2
+  };
+  std::map<vm::Cell::Hash, CellStatus> cells_;
+  td::uint64 proof_size_ = 0;
+};
+
 struct CellSerializationInfo {
   bool special;
   Cell::LevelMask level_mask;

crypto/vm/cells/CellUsageTree.cpp

@@ -22,12 +22,12 @@ namespace vm {
 //
 // CellUsageTree::NodePtr
 //
-bool CellUsageTree::NodePtr::on_load() const {
+bool CellUsageTree::NodePtr::on_load(const td::Ref<vm::DataCell>& cell) const {
   auto tree = tree_weak_.lock();
   if (!tree) {
     return false;
   }
-  tree->on_load(node_id_);
+  tree->on_load(node_id_, cell);
   return true;
 }
 
@@ -111,8 +111,14 @@ void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) {
   use_mark_ = use_mark;
 }
 
-void CellUsageTree::on_load(NodeId node_id) {
+void CellUsageTree::on_load(NodeId node_id, const td::Ref<vm::DataCell>& cell) {
+  if (nodes_[node_id].is_loaded) {
+    return;
+  }
   nodes_[node_id].is_loaded = true;
+  if (cell_load_callback_) {
+    cell_load_callback_(cell);
+  }
 }
 
 CellUsageTree::NodeId CellUsageTree::create_child(NodeId node_id, unsigned ref_id) {

crypto/vm/cells/CellUsageTree.h

@@ -22,8 +22,12 @@
 
 #include "td/utils/int_types.h"
 #include "td/utils/logging.h"
+#include <functional>
 
 namespace vm {
+
+class DataCell;
+
 class CellUsageTree : public std::enable_shared_from_this<CellUsageTree> {
  public:
   using NodeId = td::uint32;
@@ -38,7 +42,7 @@ class CellUsageTree : public std::enable_shared_from_this<CellUsageTree> {
       return node_id_ == 0 || tree_weak_.expired();
     }
 
-    bool on_load() const;
+    bool on_load(const td::Ref<vm::DataCell>& cell) const;
     NodePtr create_child(unsigned ref_id) const;
     bool mark_path(CellUsageTree* master_tree) const;
     bool is_from_tree(const CellUsageTree* master_tree) const;
@@ -59,6 +63,10 @@ class CellUsageTree : public std::enable_shared_from_this<CellUsageTree> {
   void set_use_mark_for_is_loaded(bool use_mark = true);
   NodeId create_child(NodeId node_id, unsigned ref_id);
 
+  void set_cell_load_callback(std::function<void(const td::Ref<vm::DataCell>&)> f) {
+    cell_load_callback_ = std::move(f);
+  }
+
  private:
   struct Node {
     bool is_loaded{false};
@@ -68,8 +76,9 @@ class CellUsageTree : public std::enable_shared_from_this<CellUsageTree> {
   };
   bool use_mark_{false};
   std::vector<Node> nodes_{2};
+  std::function<void(const td::Ref<vm::DataCell>&)> cell_load_callback_;
 
-  void on_load(NodeId node_id);
+  void on_load(NodeId node_id, const td::Ref<vm::DataCell>& cell);
   NodeId create_node(NodeId parent);
 };
 }  // namespace vm

crypto/vm/cells/MerkleProof.h

@@ -66,6 +66,10 @@ class MerkleProofBuilder {
   td::Result<Ref<Cell>> extract_proof() const;
   bool extract_proof_to(Ref<Cell> &proof_root) const;
   td::Result<td::BufferSlice> extract_proof_boc() const;
+
+  void set_cell_load_callback(std::function<void(const td::Ref<vm::DataCell>&)> f) {
+    usage_tree->set_cell_load_callback(std::move(f));
+  }
 };
 
 }  // namespace vm

crypto/vm/cells/UsageCell.h

@@ -39,7 +39,7 @@ class UsageCell : public Cell {
   // load interface
   td::Result<LoadedCell> load_cell() const override {
     TRY_RESULT(loaded_cell, cell_->load_cell());
-    if (tree_node_.on_load()) {
+    if (tree_node_.on_load(loaded_cell.data_cell)) {
       CHECK(loaded_cell.tree_node.empty());
       loaded_cell.tree_node = tree_node_;
     }