mirror of
				https://github.com/ton-blockchain/ton
				synced 2025-03-09 15:40:10 +00:00 
			
		
		
		
	Merge branch 'testnet' into block-generation
This commit is contained in:
		
						commit
						eb41e1a10e
					
				
					 10 changed files with 165 additions and 148 deletions
				
			
		| 
						 | 
				
			
			@ -51,32 +51,6 @@
 | 
			
		|||
#include "vm/vm.h"
 | 
			
		||||
#include "vm/cp0.h"
 | 
			
		||||
 | 
			
		||||
namespace {
 | 
			
		||||
 | 
			
		||||
td::Ref<vm::Tuple> prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, td::Ref<vm::CellSlice> my_addr,
 | 
			
		||||
                                 const block::CurrencyCollection &balance) {
 | 
			
		||||
  td::BitArray<256> rand_seed;
 | 
			
		||||
  td::RefInt256 rand_seed_int{true};
 | 
			
		||||
  td::Random::secure_bytes(rand_seed.as_slice());
 | 
			
		||||
  if (!rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false)) {
 | 
			
		||||
    return {};
 | 
			
		||||
  }
 | 
			
		||||
  auto tuple = vm::make_tuple_ref(td::make_refint(0x076ef1ea),  // [ magic:0x076ef1ea
 | 
			
		||||
                                  td::make_refint(0),           //   actions:Integer
 | 
			
		||||
                                  td::make_refint(0),           //   msgs_sent:Integer
 | 
			
		||||
                                  td::make_refint(now),         //   unixtime:Integer
 | 
			
		||||
                                  td::make_refint(lt),          //   block_lt:Integer
 | 
			
		||||
                                  td::make_refint(lt),          //   trans_lt:Integer
 | 
			
		||||
                                  std::move(rand_seed_int),     //   rand_seed:Integer
 | 
			
		||||
                                  balance.as_vm_tuple(),        //   balance_remaining:[Integer (Maybe Cell)]
 | 
			
		||||
                                  my_addr,                      //  myself:MsgAddressInt
 | 
			
		||||
                                  vm::StackEntry());            //  global_config:(Maybe Cell) ] = SmartContractInfo;
 | 
			
		||||
  LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string();
 | 
			
		||||
  return vm::make_tuple_ref(std::move(tuple));
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
}  // namespace
 | 
			
		||||
 | 
			
		||||
td::Result<ton::BlockIdExt> parse_block_id(std::map<std::string, std::string> &opts, bool allow_empty) {
 | 
			
		||||
  if (allow_empty) {
 | 
			
		||||
    if (opts.count("workchain") == 0 && opts.count("shard") == 0 && opts.count("seqno") == 0) {
 | 
			
		||||
| 
						 | 
				
			
			@ -1335,110 +1309,69 @@ void HttpQueryRunMethod::start_up_query() {
 | 
			
		|||
    if (R.is_error()) {
 | 
			
		||||
      td::actor::send_closure(SelfId, &HttpQueryRunMethod::abort_query, R.move_as_error_prefix("litequery failed: "));
 | 
			
		||||
    } else {
 | 
			
		||||
      td::actor::send_closure(SelfId, &HttpQueryRunMethod::got_account, R.move_as_ok());
 | 
			
		||||
      td::actor::send_closure(SelfId, &HttpQueryRunMethod::got_result, R.move_as_ok());
 | 
			
		||||
    }
 | 
			
		||||
  });
 | 
			
		||||
 | 
			
		||||
  auto a = ton::create_tl_object<ton::lite_api::liteServer_accountId>(addr_.workchain, addr_.addr);
 | 
			
		||||
  auto query = ton::create_tl_object<ton::lite_api::liteServer_getAccountState>(ton::create_tl_lite_block_id(block_id_),
 | 
			
		||||
                                                                                std::move(a));
 | 
			
		||||
  td::int64 method_id = (td::crc16(td::Slice{method_name_}) & 0xffff) | 0x10000;
 | 
			
		||||
 | 
			
		||||
  // serialize params
 | 
			
		||||
  vm::CellBuilder cb;
 | 
			
		||||
  td::Ref<vm::Cell> cell;
 | 
			
		||||
  if (!(vm::Stack{params_}.serialize(cb) && cb.finalize_to(cell))) {
 | 
			
		||||
    return abort_query(td::Status::Error("cannot serialize stack with get-method parameters"));
 | 
			
		||||
  }
 | 
			
		||||
  auto params_serialized = vm::std_boc_serialize(std::move(cell));
 | 
			
		||||
  if (params_serialized.is_error()) {
 | 
			
		||||
    return abort_query(params_serialized.move_as_error_prefix("cannot serialize stack with get-method parameters : "));
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  auto query = ton::create_tl_object<ton::lite_api::liteServer_runSmcMethod>(
 | 
			
		||||
      0x17, ton::create_tl_lite_block_id(block_id_), std::move(a), method_id, params_serialized.move_as_ok());
 | 
			
		||||
  td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query,
 | 
			
		||||
                          ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P));
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
void HttpQueryRunMethod::got_account(td::BufferSlice data) {
 | 
			
		||||
  auto F = ton::fetch_tl_object<ton::lite_api::liteServer_accountState>(std::move(data), true);
 | 
			
		||||
void HttpQueryRunMethod::got_result(td::BufferSlice data) {
 | 
			
		||||
  auto F = ton::fetch_tl_object<ton::lite_api::liteServer_runMethodResult>(std::move(data), true);
 | 
			
		||||
  if (F.is_error()) {
 | 
			
		||||
    abort_query(F.move_as_error());
 | 
			
		||||
    return;
 | 
			
		||||
    return abort_query(F.move_as_error());
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  auto f = F.move_as_ok();
 | 
			
		||||
  data_ = std::move(f->state_);
 | 
			
		||||
  proof_ = std::move(f->proof_);
 | 
			
		||||
  shard_proof_ = std::move(f->shard_proof_);
 | 
			
		||||
  block_id_ = ton::create_block_id(f->id_);
 | 
			
		||||
  res_block_id_ = ton::create_block_id(f->shardblk_);
 | 
			
		||||
 | 
			
		||||
  finish_query();
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
void HttpQueryRunMethod::finish_query() {
 | 
			
		||||
  if (promise_) {
 | 
			
		||||
  auto page = [&]() -> std::string {
 | 
			
		||||
    HttpAnswer A{"account", prefix_};
 | 
			
		||||
    A.set_account_id(addr_);
 | 
			
		||||
      A.set_block_id(res_block_id_);
 | 
			
		||||
 | 
			
		||||
      block::AccountState account_state;
 | 
			
		||||
      account_state.blk = block_id_;
 | 
			
		||||
      account_state.shard_blk = res_block_id_;
 | 
			
		||||
      account_state.shard_proof = std::move(shard_proof_);
 | 
			
		||||
      account_state.proof = std::move(proof_);
 | 
			
		||||
      account_state.state = std::move(data_);
 | 
			
		||||
      auto r_info = account_state.validate(block_id_, addr_);
 | 
			
		||||
      if (r_info.is_error()) {
 | 
			
		||||
        A.abort(r_info.move_as_error());
 | 
			
		||||
        return A.finish();
 | 
			
		||||
      }
 | 
			
		||||
      auto info = r_info.move_as_ok();
 | 
			
		||||
      if (info.root.is_null()) {
 | 
			
		||||
        A.abort(PSTRING() << "account state of " << addr_ << " is empty (cannot run method `" << method_name_ << "`)");
 | 
			
		||||
        return A.finish();
 | 
			
		||||
      }
 | 
			
		||||
      block::gen::Account::Record_account acc;
 | 
			
		||||
      block::gen::AccountStorage::Record store;
 | 
			
		||||
      block::CurrencyCollection balance;
 | 
			
		||||
      if (!(tlb::unpack_cell(info.root, acc) && tlb::csr_unpack(acc.storage, store) &&
 | 
			
		||||
            balance.validate_unpack(store.balance))) {
 | 
			
		||||
        A.abort("error unpacking account state");
 | 
			
		||||
        return A.finish();
 | 
			
		||||
      }
 | 
			
		||||
      int tag = block::gen::t_AccountState.get_tag(*store.state);
 | 
			
		||||
      switch (tag) {
 | 
			
		||||
        case block::gen::AccountState::account_uninit:
 | 
			
		||||
          A.abort(PSTRING() << "account " << addr_ << " not initialized yet (cannot run any methods)");
 | 
			
		||||
          return A.finish();
 | 
			
		||||
        case block::gen::AccountState::account_frozen:
 | 
			
		||||
          A.abort(PSTRING() << "account " << addr_ << " frozen (cannot run any methods)");
 | 
			
		||||
    A.set_block_id(ton::create_block_id(f->id_));
 | 
			
		||||
    if (f->exit_code_ != 0) {
 | 
			
		||||
      A.abort(PSTRING() << "VM terminated with error code " << f->exit_code_);
 | 
			
		||||
      return A.finish();
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
      CHECK(store.state.write().fetch_ulong(1) == 1);  // account_init$1 _:StateInit = AccountState;
 | 
			
		||||
      block::gen::StateInit::Record state_init;
 | 
			
		||||
      CHECK(tlb::csr_unpack(store.state, state_init));
 | 
			
		||||
      auto code = state_init.code->prefetch_ref();
 | 
			
		||||
      auto data = state_init.data->prefetch_ref();
 | 
			
		||||
      auto stack = td::make_ref<vm::Stack>(std::move(params_));
 | 
			
		||||
      td::int64 method_id = (td::crc16(td::Slice{method_name_}) & 0xffff) | 0x10000;
 | 
			
		||||
      stack.write().push_smallint(method_id);
 | 
			
		||||
      long long gas_limit = vm::GasLimits::infty;
 | 
			
		||||
      // OstreamLogger ostream_logger(ctx.error_stream);
 | 
			
		||||
      // auto log = create_vm_log(ctx.error_stream ? &ostream_logger : nullptr);
 | 
			
		||||
      vm::GasLimits gas{gas_limit};
 | 
			
		||||
      LOG(DEBUG) << "creating VM";
 | 
			
		||||
      vm::VmState vm{code, std::move(stack), gas, 1, data, vm::VmLog()};
 | 
			
		||||
      vm.set_c7(prepare_vm_c7(info.gen_utime, info.gen_lt, acc.addr, balance));  // tuple with SmartContractInfo
 | 
			
		||||
      // vm.incr_stack_trace(1);    // enable stack dump after each step
 | 
			
		||||
      int exit_code = ~vm.run();
 | 
			
		||||
      if (exit_code != 0) {
 | 
			
		||||
        A.abort(PSTRING() << "VM terminated with error code " << exit_code);
 | 
			
		||||
        return A.finish();
 | 
			
		||||
      }
 | 
			
		||||
      stack = vm.get_stack_ref();
 | 
			
		||||
      {
 | 
			
		||||
    std::ostringstream os;
 | 
			
		||||
    os << "result: ";
 | 
			
		||||
        stack->dump(os, 3);
 | 
			
		||||
 | 
			
		||||
        A << HttpAnswer::CodeBlock{os.str()};
 | 
			
		||||
    if (f->result_.empty()) {
 | 
			
		||||
      os << "<none>";
 | 
			
		||||
    } else {
 | 
			
		||||
      auto r_cell = vm::std_boc_deserialize(f->result_);
 | 
			
		||||
      if (r_cell.is_error()) {
 | 
			
		||||
        A.abort(PSTRING() << "cannot deserialize VM result boc: " << r_cell.move_as_error());
 | 
			
		||||
        return A.finish();
 | 
			
		||||
      }
 | 
			
		||||
 | 
			
		||||
      auto cs = vm::load_cell_slice(r_cell.move_as_ok());
 | 
			
		||||
      td::Ref<vm::Stack> stack;
 | 
			
		||||
      if (!(vm::Stack::deserialize_to(cs, stack, 0) && cs.empty_ext())) {
 | 
			
		||||
        A.abort("VM result boc cannot be deserialized");
 | 
			
		||||
        return A.finish();
 | 
			
		||||
      }
 | 
			
		||||
      stack->dump(os, 3);
 | 
			
		||||
    }
 | 
			
		||||
    A << HttpAnswer::CodeBlock{os.str()};
 | 
			
		||||
    return A.finish();
 | 
			
		||||
  }();
 | 
			
		||||
  auto R = MHD_create_response_from_buffer(page.length(), const_cast<char *>(page.c_str()), MHD_RESPMEM_MUST_COPY);
 | 
			
		||||
  MHD_add_response_header(R, "Content-Type", "text/html");
 | 
			
		||||
  promise_.set_value(std::move(R));
 | 
			
		||||
  }
 | 
			
		||||
  stop();
 | 
			
		||||
}
 | 
			
		||||
HttpQueryStatus::HttpQueryStatus(std::string prefix, td::Promise<MHD_Response *> promise)
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -311,22 +311,14 @@ class HttpQueryRunMethod : public HttpQueryCommon {
 | 
			
		|||
                     std::vector<vm::StackEntry> params, std::string prefix, td::Promise<MHD_Response *> promise);
 | 
			
		||||
  HttpQueryRunMethod(std::map<std::string, std::string> opts, std::string prefix, td::Promise<MHD_Response *> promise);
 | 
			
		||||
 | 
			
		||||
  void finish_query();
 | 
			
		||||
 | 
			
		||||
  void start_up_query() override;
 | 
			
		||||
  void got_account(td::BufferSlice result);
 | 
			
		||||
  void got_result(td::BufferSlice result);
 | 
			
		||||
 | 
			
		||||
 private:
 | 
			
		||||
  ton::BlockIdExt block_id_;
 | 
			
		||||
  block::StdAddress addr_;
 | 
			
		||||
 | 
			
		||||
  std::string method_name_;
 | 
			
		||||
  std::vector<vm::StackEntry> params_;
 | 
			
		||||
 | 
			
		||||
  td::BufferSlice data_;
 | 
			
		||||
  td::BufferSlice proof_;
 | 
			
		||||
  td::BufferSlice shard_proof_;
 | 
			
		||||
  ton::BlockIdExt res_block_id_;
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
class HttpQueryStatus : public HttpQueryCommon {
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -19,6 +19,7 @@
 | 
			
		|||
#pragma once
 | 
			
		||||
#include "td/utils/Status.h"
 | 
			
		||||
#include "td/utils/logging.h"
 | 
			
		||||
#include <functional>
 | 
			
		||||
namespace td {
 | 
			
		||||
class KeyValueReader {
 | 
			
		||||
 public:
 | 
			
		||||
| 
						 | 
				
			
			@ -27,6 +28,9 @@ class KeyValueReader {
 | 
			
		|||
 | 
			
		||||
  virtual Result<GetStatus> get(Slice key, std::string &value) = 0;
 | 
			
		||||
  virtual Result<size_t> count(Slice prefix) = 0;
 | 
			
		||||
  virtual Status for_each(std::function<Status(Slice, Slice)> f) {
 | 
			
		||||
    return Status::Error("for_each is not supported");
 | 
			
		||||
  }
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
class PrefixedKeyValueReader : public KeyValueReader {
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -56,7 +56,7 @@ RocksDb::~RocksDb() {
 | 
			
		|||
}
 | 
			
		||||
 | 
			
		||||
RocksDb RocksDb::clone() const {
 | 
			
		||||
  return RocksDb{db_, statistics_};
 | 
			
		||||
  return RocksDb{db_, options_};
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
 | 
			
		||||
| 
						 | 
				
			
			@ -64,16 +64,16 @@ Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
 | 
			
		|||
  {
 | 
			
		||||
    rocksdb::Options db_options;
 | 
			
		||||
 | 
			
		||||
    static auto cache = rocksdb::NewLRUCache(1 << 30);
 | 
			
		||||
    static auto default_cache = rocksdb::NewLRUCache(1 << 30);
 | 
			
		||||
    if (options.block_cache == nullptr) {
 | 
			
		||||
      options.block_cache = default_cache;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    rocksdb::BlockBasedTableOptions table_options;
 | 
			
		||||
    if (options.block_cache_size) {
 | 
			
		||||
      table_options.block_cache = rocksdb::NewLRUCache(options.block_cache_size.value());
 | 
			
		||||
    } else {
 | 
			
		||||
      table_options.block_cache = cache;
 | 
			
		||||
    }
 | 
			
		||||
    table_options.block_cache = options.block_cache;
 | 
			
		||||
    db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options));
 | 
			
		||||
 | 
			
		||||
    db_options.use_direct_reads = options.use_direct_reads;
 | 
			
		||||
    db_options.manual_wal_flush = true;
 | 
			
		||||
    db_options.create_if_missing = true;
 | 
			
		||||
    db_options.max_background_compactions = 4;
 | 
			
		||||
| 
						 | 
				
			
			@ -94,7 +94,7 @@ Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
 | 
			
		|||
    // default column family
 | 
			
		||||
    delete handles[0];
 | 
			
		||||
  }
 | 
			
		||||
  return RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB>(db), std::move(options.statistics));
 | 
			
		||||
  return RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB>(db), std::move(options));
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
std::shared_ptr<rocksdb::Statistics> RocksDb::create_statistics() {
 | 
			
		||||
| 
						 | 
				
			
			@ -109,6 +109,10 @@ void RocksDb::reset_statistics(const std::shared_ptr<rocksdb::Statistics> statis
 | 
			
		|||
  statistics->Reset();
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
std::shared_ptr<rocksdb::Cache> RocksDb::create_cache(size_t capacity) {
 | 
			
		||||
  return rocksdb::NewLRUCache(capacity);
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
std::unique_ptr<KeyValueReader> RocksDb::snapshot() {
 | 
			
		||||
  auto res = std::make_unique<RocksDb>(clone());
 | 
			
		||||
  res->begin_snapshot().ensure();
 | 
			
		||||
| 
						 | 
				
			
			@ -120,7 +124,6 @@ std::string RocksDb::stats() const {
 | 
			
		|||
  db_->GetProperty("rocksdb.stats", &out);
 | 
			
		||||
  //db_->GetProperty("rocksdb.cur-size-all-mem-tables", &out);
 | 
			
		||||
  return out;
 | 
			
		||||
  return statistics_->ToString();
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
Result<RocksDb::GetStatus> RocksDb::get(Slice key, std::string &value) {
 | 
			
		||||
| 
						 | 
				
			
			@ -187,6 +190,28 @@ Result<size_t> RocksDb::count(Slice prefix) {
 | 
			
		|||
  return res;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
Status RocksDb::for_each(std::function<Status(Slice, Slice)> f) {
 | 
			
		||||
  rocksdb::ReadOptions options;
 | 
			
		||||
  options.snapshot = snapshot_.get();
 | 
			
		||||
  std::unique_ptr<rocksdb::Iterator> iterator;
 | 
			
		||||
  if (snapshot_ || !transaction_) {
 | 
			
		||||
    iterator.reset(db_->NewIterator(options));
 | 
			
		||||
  } else {
 | 
			
		||||
    iterator.reset(transaction_->GetIterator(options));
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  iterator->SeekToFirst();
 | 
			
		||||
  for (; iterator->Valid(); iterator->Next()) {
 | 
			
		||||
    auto key = from_rocksdb(iterator->key());
 | 
			
		||||
    auto value = from_rocksdb(iterator->value());
 | 
			
		||||
    TRY_STATUS(f(key, value));
 | 
			
		||||
  }
 | 
			
		||||
  if (!iterator->status().ok()) {
 | 
			
		||||
    return from_rocksdb(iterator->status());
 | 
			
		||||
  }
 | 
			
		||||
  return Status::OK();
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
Status RocksDb::begin_write_batch() {
 | 
			
		||||
  CHECK(!transaction_);
 | 
			
		||||
  write_batch_ = std::make_unique<rocksdb::WriteBatch>();
 | 
			
		||||
| 
						 | 
				
			
			@ -243,7 +268,7 @@ Status RocksDb::end_snapshot() {
 | 
			
		|||
  return td::Status::OK();
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
RocksDb::RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db, std::shared_ptr<rocksdb::Statistics> statistics)
 | 
			
		||||
    : db_(std::move(db)), statistics_(std::move(statistics)) {
 | 
			
		||||
RocksDb::RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db, RocksDbOptions options)
 | 
			
		||||
    : db_(std::move(db)), options_(options) {
 | 
			
		||||
}
 | 
			
		||||
}  // namespace td
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -27,6 +27,7 @@
 | 
			
		|||
#include "td/utils/optional.h"
 | 
			
		||||
 | 
			
		||||
namespace rocksdb {
 | 
			
		||||
class Cache;
 | 
			
		||||
class OptimisticTransactionDB;
 | 
			
		||||
class Transaction;
 | 
			
		||||
class WriteBatch;
 | 
			
		||||
| 
						 | 
				
			
			@ -38,7 +39,8 @@ namespace td {
 | 
			
		|||
 | 
			
		||||
struct RocksDbOptions {
 | 
			
		||||
  std::shared_ptr<rocksdb::Statistics> statistics = nullptr;
 | 
			
		||||
  optional<uint64> block_cache_size;  // Default - one 1G cache for all RocksDb
 | 
			
		||||
  std::shared_ptr<rocksdb::Cache> block_cache;  // Default - one 1G cache for all RocksDb
 | 
			
		||||
  bool use_direct_reads = false;
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
class RocksDb : public KeyValue {
 | 
			
		||||
| 
						 | 
				
			
			@ -51,6 +53,7 @@ class RocksDb : public KeyValue {
 | 
			
		|||
  Status set(Slice key, Slice value) override;
 | 
			
		||||
  Status erase(Slice key) override;
 | 
			
		||||
  Result<size_t> count(Slice prefix) override;
 | 
			
		||||
  Status for_each(std::function<Status(Slice, Slice)> f) override;
 | 
			
		||||
 | 
			
		||||
  Status begin_write_batch() override;
 | 
			
		||||
  Status commit_write_batch() override;
 | 
			
		||||
| 
						 | 
				
			
			@ -71,6 +74,8 @@ class RocksDb : public KeyValue {
 | 
			
		|||
  static std::string statistics_to_string(const std::shared_ptr<rocksdb::Statistics> statistics);
 | 
			
		||||
  static void reset_statistics(const std::shared_ptr<rocksdb::Statistics> statistics);
 | 
			
		||||
 | 
			
		||||
  static std::shared_ptr<rocksdb::Cache> create_cache(size_t capacity);
 | 
			
		||||
 | 
			
		||||
  RocksDb(RocksDb &&);
 | 
			
		||||
  RocksDb &operator=(RocksDb &&);
 | 
			
		||||
  ~RocksDb();
 | 
			
		||||
| 
						 | 
				
			
			@ -81,7 +86,7 @@ class RocksDb : public KeyValue {
 | 
			
		|||
 | 
			
		||||
 private:
 | 
			
		||||
  std::shared_ptr<rocksdb::OptimisticTransactionDB> db_;
 | 
			
		||||
  std::shared_ptr<rocksdb::Statistics> statistics_;
 | 
			
		||||
  RocksDbOptions options_;
 | 
			
		||||
 | 
			
		||||
  std::unique_ptr<rocksdb::Transaction> transaction_;
 | 
			
		||||
  std::unique_ptr<rocksdb::WriteBatch> write_batch_;
 | 
			
		||||
| 
						 | 
				
			
			@ -94,7 +99,6 @@ class RocksDb : public KeyValue {
 | 
			
		|||
  };
 | 
			
		||||
  std::unique_ptr<const rocksdb::Snapshot, UnreachableDeleter> snapshot_;
 | 
			
		||||
 | 
			
		||||
  explicit RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db,
 | 
			
		||||
                   std::shared_ptr<rocksdb::Statistics> statistics);
 | 
			
		||||
  explicit RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db, RocksDbOptions options);
 | 
			
		||||
};
 | 
			
		||||
}  // namespace td
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -1432,6 +1432,11 @@ td::Status ValidatorEngine::load_global_config() {
 | 
			
		|||
  if (celldb_cache_size_) {
 | 
			
		||||
    validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value());
 | 
			
		||||
  }
 | 
			
		||||
  if (!celldb_cache_size_ || celldb_cache_size_.value() < (30ULL << 30)) {
 | 
			
		||||
    celldb_direct_io_ = false;
 | 
			
		||||
  }
 | 
			
		||||
  validator_options_.write().set_celldb_direct_io(celldb_direct_io_);
 | 
			
		||||
  validator_options_.write().set_celldb_preload_all(celldb_preload_all_);
 | 
			
		||||
  if (catchain_max_block_delay_) {
 | 
			
		||||
    validator_options_.write().set_catchain_max_block_delay(catchain_max_block_delay_.value());
 | 
			
		||||
  }
 | 
			
		||||
| 
						 | 
				
			
			@ -4285,8 +4290,7 @@ int main(int argc, char *argv[]) {
 | 
			
		|||
    acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); });
 | 
			
		||||
  });
 | 
			
		||||
  p.add_checked_option(
 | 
			
		||||
      '\0', "celldb-cache-size",
 | 
			
		||||
      "block cache size for RocksDb in CellDb, in bytes (default: 1G cache shared by archive DB)",
 | 
			
		||||
      '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 50G)",
 | 
			
		||||
      [&](td::Slice s) -> td::Status {
 | 
			
		||||
        TRY_RESULT(v, td::to_integer_safe<td::uint64>(s));
 | 
			
		||||
        if (v == 0) {
 | 
			
		||||
| 
						 | 
				
			
			@ -4295,6 +4299,13 @@ int main(int argc, char *argv[]) {
 | 
			
		|||
        acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); });
 | 
			
		||||
        return td::Status::OK();
 | 
			
		||||
      });
 | 
			
		||||
  p.add_option(
 | 
			
		||||
      '\0', "celldb-no-direct-io", "disable direct I/O mode for RocksDb in CellDb (forced when celldb cache is < 30G)",
 | 
			
		||||
      [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, false); }); });
 | 
			
		||||
  p.add_option(
 | 
			
		||||
      '\0', "celldb-no-preload-all",
 | 
			
		||||
      "disable preloading all cells from CellDb on startup (enabled by default)",
 | 
			
		||||
      [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, false); }); });
 | 
			
		||||
  p.add_checked_option(
 | 
			
		||||
      '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)",
 | 
			
		||||
      [&](td::Slice s) -> td::Status {
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -223,7 +223,9 @@ class ValidatorEngine : public td::actor::Actor {
 | 
			
		|||
  double archive_preload_period_ = 0.0;
 | 
			
		||||
  bool disable_rocksdb_stats_ = false;
 | 
			
		||||
  bool nonfinal_ls_queries_enabled_ = false;
 | 
			
		||||
  td::optional<td::uint64> celldb_cache_size_;
 | 
			
		||||
  td::optional<td::uint64> celldb_cache_size_ = 50LL << 30;
 | 
			
		||||
  bool celldb_direct_io_ = true;
 | 
			
		||||
  bool celldb_preload_all_ = true;
 | 
			
		||||
  td::optional<double> catchain_max_block_delay_;
 | 
			
		||||
  bool read_config_ = false;
 | 
			
		||||
  bool started_keyring_ = false;
 | 
			
		||||
| 
						 | 
				
			
			@ -303,6 +305,12 @@ class ValidatorEngine : public td::actor::Actor {
 | 
			
		|||
  void set_celldb_cache_size(td::uint64 value) {
 | 
			
		||||
    celldb_cache_size_ = value;
 | 
			
		||||
  }
 | 
			
		||||
  void set_celldb_direct_io(bool value) {
 | 
			
		||||
    celldb_direct_io_ = value;
 | 
			
		||||
  }
 | 
			
		||||
  void set_celldb_preload_all(bool value) {
 | 
			
		||||
    celldb_preload_all_ = value;
 | 
			
		||||
  }
 | 
			
		||||
  void set_catchain_max_block_delay(double value) {
 | 
			
		||||
    catchain_max_block_delay_ = value;
 | 
			
		||||
  }
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -91,9 +91,10 @@ void CellDbIn::start_up() {
 | 
			
		|||
  td::RocksDbOptions db_options;
 | 
			
		||||
  db_options.statistics = statistics_;
 | 
			
		||||
  if (opts_->get_celldb_cache_size()) {
 | 
			
		||||
    db_options.block_cache_size = opts_->get_celldb_cache_size().value();
 | 
			
		||||
    LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(db_options.block_cache_size.value());
 | 
			
		||||
    db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value());
 | 
			
		||||
    LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value());
 | 
			
		||||
  }
 | 
			
		||||
  db_options.use_direct_reads = opts_->get_celldb_direct_io();
 | 
			
		||||
  cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_, std::move(db_options)).move_as_ok());
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			@ -111,6 +112,27 @@ void CellDbIn::start_up() {
 | 
			
		|||
    set_block(empty, std::move(e));
 | 
			
		||||
    cell_db_->commit_write_batch().ensure();
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  if (opts_->get_celldb_preload_all()) {
 | 
			
		||||
    // Iterate whole DB in a separate thread
 | 
			
		||||
    delay_action([snapshot = cell_db_->snapshot()]() {
 | 
			
		||||
      LOG(WARNING) << "CellDb: pre-loading all keys";
 | 
			
		||||
      td::uint64 total = 0;
 | 
			
		||||
      td::Timer timer;
 | 
			
		||||
      auto S = snapshot->for_each([&](td::Slice, td::Slice) {
 | 
			
		||||
        ++total;
 | 
			
		||||
        if (total % 1000000 == 0) {
 | 
			
		||||
          LOG(INFO) << "CellDb: iterated " << total << " keys";
 | 
			
		||||
        }
 | 
			
		||||
        return td::Status::OK();
 | 
			
		||||
      });
 | 
			
		||||
      if (S.is_error()) {
 | 
			
		||||
        LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error();
 | 
			
		||||
      } else {
 | 
			
		||||
      LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s";
 | 
			
		||||
      }
 | 
			
		||||
    }, td::Timestamp::now());
 | 
			
		||||
  }
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
void CellDbIn::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -130,6 +130,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
 | 
			
		|||
  td::optional<td::uint64> get_celldb_cache_size() const override {
 | 
			
		||||
    return celldb_cache_size_;
 | 
			
		||||
  }
 | 
			
		||||
  bool get_celldb_direct_io() const override {
 | 
			
		||||
    return celldb_direct_io_;
 | 
			
		||||
  }
 | 
			
		||||
  bool get_celldb_preload_all() const override {
 | 
			
		||||
    return celldb_preload_all_;
 | 
			
		||||
  }
 | 
			
		||||
  td::optional<double> get_catchain_max_block_delay() const override {
 | 
			
		||||
    return catchain_max_block_delay_;
 | 
			
		||||
  }
 | 
			
		||||
| 
						 | 
				
			
			@ -207,6 +213,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
 | 
			
		|||
  void set_celldb_cache_size(td::uint64 value) override {
 | 
			
		||||
    celldb_cache_size_ = value;
 | 
			
		||||
  }
 | 
			
		||||
  void set_celldb_direct_io(bool value) override {
 | 
			
		||||
    celldb_direct_io_ = value;
 | 
			
		||||
  }
 | 
			
		||||
  void set_celldb_preload_all(bool value) override {
 | 
			
		||||
    celldb_preload_all_ = value;
 | 
			
		||||
  }
 | 
			
		||||
  void set_catchain_max_block_delay(double value) override {
 | 
			
		||||
    catchain_max_block_delay_ = value;
 | 
			
		||||
  }
 | 
			
		||||
| 
						 | 
				
			
			@ -259,6 +271,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
 | 
			
		|||
  bool disable_rocksdb_stats_;
 | 
			
		||||
  bool nonfinal_ls_queries_enabled_ = false;
 | 
			
		||||
  td::optional<td::uint64> celldb_cache_size_;
 | 
			
		||||
  bool celldb_direct_io_ = false;
 | 
			
		||||
  bool celldb_preload_all_ = false;
 | 
			
		||||
  td::optional<double> catchain_max_block_delay_;
 | 
			
		||||
  ValidatorMode validator_mode_ = validator_normal;
 | 
			
		||||
};
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -87,6 +87,8 @@ struct ValidatorManagerOptions : public td::CntObject {
 | 
			
		|||
  virtual bool get_disable_rocksdb_stats() const = 0;
 | 
			
		||||
  virtual bool nonfinal_ls_queries_enabled() const = 0;
 | 
			
		||||
  virtual td::optional<td::uint64> get_celldb_cache_size() const = 0;
 | 
			
		||||
  virtual bool get_celldb_direct_io() const = 0;
 | 
			
		||||
  virtual bool get_celldb_preload_all() const = 0;
 | 
			
		||||
  virtual td::optional<double> get_catchain_max_block_delay() const = 0;
 | 
			
		||||
  virtual ValidatorMode validator_mode() const = 0;
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			@ -113,6 +115,8 @@ struct ValidatorManagerOptions : public td::CntObject {
 | 
			
		|||
  virtual void set_disable_rocksdb_stats(bool value) = 0;
 | 
			
		||||
  virtual void set_nonfinal_ls_queries_enabled(bool value) = 0;
 | 
			
		||||
  virtual void set_celldb_cache_size(td::uint64 value) = 0;
 | 
			
		||||
  virtual void set_celldb_direct_io(bool value) = 0;
 | 
			
		||||
  virtual void set_celldb_preload_all(bool value) = 0;
 | 
			
		||||
  virtual void set_catchain_max_block_delay(double value) = 0;
 | 
			
		||||
  virtual void set_validator_mode(ValidatorMode value) = 0;
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue