liteserver: bugfix

liteserver/liteclient: fixed bug in proof
validator: added stats
smartcontracts: updates

validator/db/filedb.cpp

@@ -353,6 +353,45 @@ FileDb::DbEntry::DbEntry(tl_object_ptr<ton_api::db_filedb_value> entry)
     , file_hash(entry->file_hash_) {
 }
 
+void FileDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
+  std::vector<std::pair<std::string, std::string>> rocksdb_stats;
+  auto stats = kv_->stats();
+  if (stats.size() == 0) {
+    promise.set_value(std::move(rocksdb_stats));
+    return;
+  }
+  size_t pos = 0;
+  while (pos < stats.size()) {
+    while (pos < stats.size() &&
+           (stats[pos] == ' ' || stats[pos] == '\n' || stats[pos] == '\r' || stats[pos] == '\t')) {
+      pos++;
+    }
+    auto p = pos;
+    if (pos == stats.size()) {
+      break;
+    }
+    while (stats[pos] != '\n' && stats[pos] != '\r' && stats[pos] != ' ' && stats[pos] != '\t' && pos < stats.size()) {
+      pos++;
+    }
+    auto name = stats.substr(p, pos - p);
+    if (stats[pos] == '\n' || pos == stats.size()) {
+      rocksdb_stats.emplace_back(name, "");
+      continue;
+    }
+    while (pos < stats.size() &&
+           (stats[pos] == ' ' || stats[pos] == '\n' || stats[pos] == '\r' || stats[pos] == '\t')) {
+      pos++;
+    }
+    p = pos;
+    while (stats[pos] != '\n' && stats[pos] != '\r' && pos < stats.size()) {
+      pos++;
+    }
+    auto value = stats.substr(p, pos - p);
+    rocksdb_stats.emplace_back(name, value);
+  }
+  promise.set_value(std::move(rocksdb_stats));
+}
+
 }  // namespace validator
 
 }  // namespace ton

validator/db/filedb.hpp

@@ -147,6 +147,8 @@ class FileDb : public td::actor::Actor {
   void load_file_slice(RefId ref_id, td::int64 offset, td::int64 max_size, td::Promise<td::BufferSlice> promise);
   void check_file(RefId ref_id, td::Promise<bool> promise);
 
+  void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise);
+
   void start_up() override;
   void alarm() override;
 

validator/db/rootdb.cpp

@@ -24,6 +24,7 @@
 #include "ton/ton-tl.hpp"
 #include "td/utils/overloaded.h"
 #include "common/checksum.h"
+#include "validator/stats-merger.h"
 
 namespace ton {
 
@@ -473,6 +474,13 @@ void RootDb::allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise<bool> p
                      }));
 }
 
+void RootDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
+  auto merger = StatsMerger::create(std::move(promise));
+
+  td::actor::send_closure(file_db_, &FileDb::prepare_stats, merger.make_promise("filedb."));
+  td::actor::send_closure(archive_db_, &FileDb::prepare_stats, merger.make_promise("archivedb."));
+}
+
 }  // namespace validator
 
 }  // namespace ton

validator/db/rootdb.hpp

@@ -110,6 +110,8 @@ class RootDb : public Db {
   void allow_block_gc(BlockIdExt block_id, td::Promise<bool> promise);
   void allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise<bool> promise);
 
+  void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) override;
+
  private:
   td::actor::ActorId<ValidatorManager> validator_manager_;
 

validator/impl/liteserver.cpp

@@ -780,11 +780,10 @@ bool LiteQuery::make_state_root_proof(Ref<vm::Cell>& proof, Ref<vm::Cell> state_
   CHECK(block_root.not_null() && state_root.not_null());
   RootHash rhash{block_root->get_hash().bits()};
   CHECK(rhash == blkid.root_hash);
-  auto usage_tree = std::make_shared<vm::CellUsageTree>();
-  auto usage_cell = vm::UsageCell::create(block_root, usage_tree->root_ptr());
+  vm::MerkleProofBuilder pb{std::move(block_root)};
   block::gen::Block::Record blk;
   block::gen::BlockInfo::Record info;
-  if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(blk.info, info))) {
+  if (!(tlb::unpack_cell(pb.root(), blk) && tlb::unpack_cell(blk.info, info))) {
     return fatal_error("cannot unpack block header");
   }
   vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update};
@@ -797,8 +796,7 @@ bool LiteQuery::make_state_root_proof(Ref<vm::Cell>& proof, Ref<vm::Cell> state_
   if (upd_hash.compare(state_hash, 256)) {
     return fatal_error("cannot construct Merkle proof for given masterchain state because of hash mismatch");
   }
-  proof = vm::MerkleProof::generate(block_root, usage_tree.get());
-  if (proof.is_null()) {
+  if (!pb.extract_proof_to(proof)) {
     return fatal_error("unknown error creating Merkle proof");
   }
   return true;
@@ -806,20 +804,17 @@ bool LiteQuery::make_state_root_proof(Ref<vm::Cell>& proof, Ref<vm::Cell> state_
 
 bool LiteQuery::make_shard_info_proof(Ref<vm::Cell>& proof, vm::CellSlice& cs, ShardIdFull shard,
                                       ShardIdFull& true_shard, Ref<vm::Cell>& leaf, bool& found, bool exact) {
-  auto state_root = mc_state_->root_cell();
-  auto usage_tree = std::make_shared<vm::CellUsageTree>();
-  auto usage_cell = vm::UsageCell::create(state_root, usage_tree->root_ptr());
+  vm::MerkleProofBuilder pb{mc_state_->root_cell()};
   block::gen::ShardStateUnsplit::Record sstate;
-  if (!(tlb::unpack_cell(usage_cell, sstate))) {
+  if (!(tlb::unpack_cell(pb.root(), sstate))) {
     return fatal_error("cannot unpack state header");
   }
-  auto shards_dict = block::ShardConfig::extract_shard_hashes_dict(usage_cell);
+  auto shards_dict = block::ShardConfig::extract_shard_hashes_dict(pb.root());
   if (!shards_dict) {
     return fatal_error("cannot extract ShardHashes from last mc state");
   }
   found = block::ShardConfig::get_shard_hash_raw_from(*shards_dict, cs, shard, true_shard, exact, &leaf);
-  proof = vm::MerkleProof::generate(state_root, usage_tree.get());
-  if (proof.is_null()) {
+  if (!pb.extract_proof_to(proof)) {
     return fatal_error("unknown error creating Merkle proof");
   }
   return true;
@@ -921,11 +916,9 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) {
   if (!make_state_root_proof(proof1)) {
     return;
   }
-  auto state_root = state_->root_cell();
-  auto usage_tree = std::make_shared<vm::CellUsageTree>();
-  auto usage_cell = vm::UsageCell::create(state_root, usage_tree->root_ptr());
+  vm::MerkleProofBuilder pb{state_->root_cell()};
   block::gen::ShardStateUnsplit::Record sstate;
-  if (!(tlb::unpack_cell(usage_cell, sstate))) {
+  if (!tlb::unpack_cell(pb.root(), sstate)) {
     fatal_error("cannot unpack state header");
     return;
   }
@@ -935,10 +928,7 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) {
   if (acc_csr.not_null()) {
     acc_root = acc_csr->prefetch_ref();
   }
-  auto proof2 = vm::MerkleProof::generate(state_root, usage_tree.get());
-  usage_tree.reset();
-  usage_cell.clear();
-  auto proof = vm::std_boc_serialize_multi({std::move(proof1), std::move(proof2)});
+  auto proof = vm::std_boc_serialize_multi({std::move(proof1), pb.extract_proof()});
   if (proof.is_error()) {
     fatal_error(proof.move_as_error());
     return;
@@ -962,23 +952,18 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) {
 void LiteQuery::continue_getOneTransaction() {
   LOG(INFO) << "completing getOneTransaction() query";
   CHECK(block_.not_null());
-  auto block_root = block_->root_cell();
-  auto usage_tree = std::make_shared<vm::CellUsageTree>();
-  auto usage_cell = vm::UsageCell::create(block_root, usage_tree->root_ptr());
-  auto trans_res = block::get_block_transaction(block_root, acc_workchain_, acc_addr_, trans_lt_);
+  vm::MerkleProofBuilder pb{block_->root_cell()};
+  auto trans_res = block::get_block_transaction(pb.root(), acc_workchain_, acc_addr_, trans_lt_);
   if (trans_res.is_error()) {
     fatal_error(trans_res.move_as_error());
     return;
   }
   auto trans_root = trans_res.move_as_ok();
-  auto proof = vm::MerkleProof::generate(block_root, usage_tree.get());
-  auto proof_boc = vm::std_boc_serialize(std::move(proof));
+  auto proof_boc = pb.extract_proof_boc();
   if (proof_boc.is_error()) {
     fatal_error(proof_boc.move_as_error());
     return;
   }
-  usage_tree.reset();
-  usage_cell.clear();
   td::BufferSlice data;
   if (trans_root.not_null()) {
     auto res = vm::std_boc_serialize(std::move(trans_root));
@@ -999,10 +984,13 @@ void LiteQuery::perform_getTransactions(WorkchainId workchain, StdSmcAddress add
                                         unsigned count) {
   LOG(INFO) << "started a getTransactions(" << workchain << ", " << addr.to_hex() << ", " << lt << ", " << hash.to_hex()
             << ", " << count << ") liteserver query";
-  if (count > 10) {
+  count = std::min(count, (unsigned)max_transaction_count);
+  /*
+  if (count > max_transaction_count) {
     fatal_error("cannot fetch more than 10 preceding transactions at one time");
     return;
   }
+  */
   if (workchain == ton::workchainInvalid) {
     fatal_error("invalid workchain specified");
     return;
@@ -1355,14 +1343,11 @@ void LiteQuery::finish_listBlockTransactions(int mode, int req_count) {
   CHECK(block_root.not_null());
   RootHash rhash{block_root->get_hash().bits()};
   CHECK(rhash == base_blk_id_.root_hash);
-  Ref<vm::Cell> usage_cell;
-  std::shared_ptr<vm::CellUsageTree> usage_tree;
+  vm::MerkleProofBuilder pb;
+  auto virt_root = block_root;
   if (mode & 32) {
     // proof requested
-    usage_tree = std::make_shared<vm::CellUsageTree>();
-    usage_cell = vm::UsageCell::create(block_root, usage_tree->root_ptr());
-  } else {
-    usage_cell = block_root;
+    virt_root = pb.init(std::move(virt_root));
   }
   if ((mode & 192) == 64) {  // reverse order, no starting point
     acc_addr_.set_ones();
@@ -1374,7 +1359,7 @@ void LiteQuery::finish_listBlockTransactions(int mode, int req_count) {
   try {
     block::gen::Block::Record blk;
     block::gen::BlockExtra::Record extra;
-    if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(std::move(blk.extra), extra))) {
+    if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(std::move(blk.extra), extra))) {
       fatal_error("cannot find account transaction data in block "s + base_blk_id_.to_str());
       return;
     }
@@ -1433,8 +1418,7 @@ void LiteQuery::finish_listBlockTransactions(int mode, int req_count) {
   td::BufferSlice proof_data;
   if (mode & 32) {
     // create proof
-    auto proof = vm::MerkleProof::generate(block_root, usage_tree.get());
-    auto proof_boc = vm::std_boc_serialize(std::move(proof));
+    auto proof_boc = pb.extract_proof_boc();
     if (proof_boc.is_error()) {
       fatal_error(proof_boc.move_as_error());
       return;

validator/impl/liteserver.hpp

@@ -57,7 +57,10 @@ class LiteQuery : public td::actor::Actor {
   std::unique_ptr<block::BlockProofChain> chain_;
 
  public:
-  enum { default_timeout_msec = 4500 };  // 4.5 seconds
+  enum {
+    default_timeout_msec = 4500,  // 4.5 seconds
+    max_transaction_count = 16    // fetch at most 16 transactions in one query
+  };
   enum {
     ls_version = 0x101,
     ls_capabilities = 3

validator/interfaces/db.h

@@ -93,6 +93,8 @@ class Db : public td::actor::Actor {
   virtual void get_async_serializer_state(td::Promise<AsyncSerializerState> promise) = 0;
 
   virtual void archive(BlockIdExt block_id, td::Promise<td::Unit> promise) = 0;
+
+  virtual void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) = 0;
 };
 
 }  // namespace validator

validator/manager.cpp

@@ -38,6 +38,8 @@
 
 #include "common/delay.h"
 
+#include "validator/stats-merger.h"
+
 namespace ton {
 
 namespace validator {
@@ -1811,6 +1813,7 @@ void ValidatorManagerImpl::advance_gc(BlockHandle handle, td::Ref<MasterchainSta
   gc_advancing_ = false;
   gc_masterchain_handle_ = std::move(handle);
   gc_masterchain_state_ = std::move(state);
+  try_advance_gc_masterchain_block();
 }
 
 void ValidatorManagerImpl::shard_client_update(BlockSeqno seqno) {
@@ -1970,21 +1973,37 @@ void ValidatorManagerImpl::send_peek_key_block_request() {
 }
 
 void ValidatorManagerImpl::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
+  auto merger = StatsMerger::create(std::move(promise));
+
+  td::actor::send_closure(db_, &Db::prepare_stats, merger.make_promise("db."));
+
   std::vector<std::pair<std::string, std::string>> vec;
   vec.emplace_back("unixtime", td::to_string(static_cast<UnixTime>(td::Clocks::system())));
-  if (!last_masterchain_block_handle_) {
-    promise.set_value(std::move(vec));
-    return;
+  if (last_masterchain_block_handle_) {
+    vec.emplace_back("masterchainblock", last_masterchain_block_id_.to_str());
+    vec.emplace_back("masterchainblocktime", td::to_string(last_masterchain_block_handle_->unix_time()));
+    vec.emplace_back("gcmasterchainblock", gc_masterchain_handle_->id().to_str());
+    vec.emplace_back("keymasterchainblock", last_key_block_handle_->id().to_str());
+    vec.emplace_back("knownkeymasterchainblock", last_known_key_block_handle_->id().to_str());
+    vec.emplace_back("rotatemasterchainblock", last_rotate_block_id_.to_str());
+    //vec.emplace_back("shardclientmasterchainseqno", td::to_string(min_confirmed_masterchain_seqno_));
+    vec.emplace_back("stateserializermasterchainseqno", td::to_string(state_serializer_masterchain_seqno_));
   }
-  vec.emplace_back("masterchainblock", last_masterchain_block_id_.to_str());
-  vec.emplace_back("masterchainblocktime", td::to_string(last_masterchain_block_handle_->unix_time()));
-  vec.emplace_back("gcmasterchainblock", gc_masterchain_handle_->id().to_str());
-  vec.emplace_back("keymasterchainblock", last_key_block_handle_->id().to_str());
-  vec.emplace_back("knownkeymasterchainblock", last_known_key_block_handle_->id().to_str());
-  vec.emplace_back("rotatemasterchainblock", last_rotate_block_id_.to_str());
-  vec.emplace_back("shardclientmasterchainseqno", td::to_string(min_confirmed_masterchain_seqno_));
-  vec.emplace_back("stateserializermasterchainseqno", td::to_string(state_serializer_masterchain_seqno_));
-  promise.set_value(std::move(vec));
+
+  if (!shard_client_.empty()) {
+    auto P = td::PromiseCreator::lambda([promise = merger.make_promise("")](td::Result<BlockSeqno> R) mutable {
+      if (R.is_error()) {
+        promise.set_error(R.move_as_error());
+        return;
+      }
+      std::vector<std::pair<std::string, std::string>> vec;
+      vec.emplace_back("shardclientmasterchainseqno", td::to_string(R.move_as_ok()));
+      promise.set_value(std::move(vec));
+    });
+    td::actor::send_closure(shard_client_, &ShardClient::get_processed_masterchain_block, std::move(P));
+  }
+
+  merger.make_promise("").set_value(std::move(vec));
 }
 
 td::actor::ActorOwn<ValidatorManagerInterface> ValidatorManagerFactory::create(

validator/stats-merger.h

@@ -0,0 +1,74 @@
+#pragma once
+
+#include "td/utils/int_types.h"
+#include "td/actor/actor.h"
+
+namespace ton {
+
+namespace validator {
+
+class StatsMerger : public td::actor::Actor {
+ public:
+  StatsMerger(td::Promise<std::vector<std::pair<std::string, std::string>>> promise)
+
+      : promise_(std::move(promise)) {
+  }
+  void start_up() override {
+    if (!pending_) {
+      finish();
+    }
+  }
+
+  void finish_subjob(td::Result<std::vector<std::pair<std::string, std::string>>> R, std::string prefix) {
+    if (R.is_ok()) {
+      auto v = R.move_as_ok();
+      for (auto &el : v) {
+        cur_.emplace_back(prefix + el.first, std::move(el.second));
+      }
+    }
+    if (--pending_ == 0) {
+      finish();
+    }
+  }
+  void inc() {
+    ++pending_;
+  }
+  void dec() {
+    if (--pending_ == 0) {
+      finish();
+    }
+  }
+  void finish() {
+    promise_.set_value(std::move(cur_));
+    stop();
+  }
+
+  struct InitGuard {
+    td::actor::ActorId<StatsMerger> merger;
+    ~InitGuard() {
+      td::actor::send_closure(merger, &StatsMerger::dec);
+    }
+    auto make_promise(std::string prefix) {
+      merger.get_actor_unsafe().inc();
+      return td::PromiseCreator::lambda(
+          [merger = merger, prefix](td::Result<std::vector<std::pair<std::string, std::string>>> R) {
+            td::actor::send_closure(merger, &StatsMerger::finish_subjob, std::move(R), std::move(prefix));
+          });
+    }
+  };
+
+  static InitGuard create(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
+    InitGuard ig;
+    ig.merger = td::actor::create_actor<StatsMerger>("m", std::move(promise)).release();
+    return ig;
+  }
+
+ private:
+  std::vector<std::pair<std::string, std::string>> cur_;
+  std::atomic<td::uint32> pending_{1};
+  td::Promise<std::vector<std::pair<std::string, std::string>>> promise_;
+};
+
+}  // namespace validator
+
+}  // namespace ton

validator/validator-options.hpp

@@ -125,7 +125,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id,
                               std::function<bool(ShardIdFull)> check_shard, bool allow_blockchain_init,
                               td::ClocksBase::Duration sync_blocks_before, td::ClocksBase::Duration block_ttl,
-                              td::ClocksBase::Duration archive_ttl, td::ClocksBase::Duration state_ttl,
+                              td::ClocksBase::Duration state_ttl, td::ClocksBase::Duration archive_ttl,
                               td::ClocksBase::Duration key_proof_ttl, bool initial_sync_disabled)
       : zero_block_id_(zero_block_id)
       , init_block_id_(init_block_id)