mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
Merge branch 'testnet' into block-generation
This commit is contained in:
commit
eb41e1a10e
10 changed files with 165 additions and 148 deletions
|
@ -51,32 +51,6 @@
|
||||||
#include "vm/vm.h"
|
#include "vm/vm.h"
|
||||||
#include "vm/cp0.h"
|
#include "vm/cp0.h"
|
||||||
|
|
||||||
namespace {
|
|
||||||
|
|
||||||
td::Ref<vm::Tuple> prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, td::Ref<vm::CellSlice> my_addr,
|
|
||||||
const block::CurrencyCollection &balance) {
|
|
||||||
td::BitArray<256> rand_seed;
|
|
||||||
td::RefInt256 rand_seed_int{true};
|
|
||||||
td::Random::secure_bytes(rand_seed.as_slice());
|
|
||||||
if (!rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false)) {
|
|
||||||
return {};
|
|
||||||
}
|
|
||||||
auto tuple = vm::make_tuple_ref(td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea
|
|
||||||
td::make_refint(0), // actions:Integer
|
|
||||||
td::make_refint(0), // msgs_sent:Integer
|
|
||||||
td::make_refint(now), // unixtime:Integer
|
|
||||||
td::make_refint(lt), // block_lt:Integer
|
|
||||||
td::make_refint(lt), // trans_lt:Integer
|
|
||||||
std::move(rand_seed_int), // rand_seed:Integer
|
|
||||||
balance.as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)]
|
|
||||||
my_addr, // myself:MsgAddressInt
|
|
||||||
vm::StackEntry()); // global_config:(Maybe Cell) ] = SmartContractInfo;
|
|
||||||
LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string();
|
|
||||||
return vm::make_tuple_ref(std::move(tuple));
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
td::Result<ton::BlockIdExt> parse_block_id(std::map<std::string, std::string> &opts, bool allow_empty) {
|
td::Result<ton::BlockIdExt> parse_block_id(std::map<std::string, std::string> &opts, bool allow_empty) {
|
||||||
if (allow_empty) {
|
if (allow_empty) {
|
||||||
if (opts.count("workchain") == 0 && opts.count("shard") == 0 && opts.count("seqno") == 0) {
|
if (opts.count("workchain") == 0 && opts.count("shard") == 0 && opts.count("seqno") == 0) {
|
||||||
|
@ -1335,110 +1309,69 @@ void HttpQueryRunMethod::start_up_query() {
|
||||||
if (R.is_error()) {
|
if (R.is_error()) {
|
||||||
td::actor::send_closure(SelfId, &HttpQueryRunMethod::abort_query, R.move_as_error_prefix("litequery failed: "));
|
td::actor::send_closure(SelfId, &HttpQueryRunMethod::abort_query, R.move_as_error_prefix("litequery failed: "));
|
||||||
} else {
|
} else {
|
||||||
td::actor::send_closure(SelfId, &HttpQueryRunMethod::got_account, R.move_as_ok());
|
td::actor::send_closure(SelfId, &HttpQueryRunMethod::got_result, R.move_as_ok());
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
auto a = ton::create_tl_object<ton::lite_api::liteServer_accountId>(addr_.workchain, addr_.addr);
|
auto a = ton::create_tl_object<ton::lite_api::liteServer_accountId>(addr_.workchain, addr_.addr);
|
||||||
auto query = ton::create_tl_object<ton::lite_api::liteServer_getAccountState>(ton::create_tl_lite_block_id(block_id_),
|
td::int64 method_id = (td::crc16(td::Slice{method_name_}) & 0xffff) | 0x10000;
|
||||||
std::move(a));
|
|
||||||
|
// serialize params
|
||||||
|
vm::CellBuilder cb;
|
||||||
|
td::Ref<vm::Cell> cell;
|
||||||
|
if (!(vm::Stack{params_}.serialize(cb) && cb.finalize_to(cell))) {
|
||||||
|
return abort_query(td::Status::Error("cannot serialize stack with get-method parameters"));
|
||||||
|
}
|
||||||
|
auto params_serialized = vm::std_boc_serialize(std::move(cell));
|
||||||
|
if (params_serialized.is_error()) {
|
||||||
|
return abort_query(params_serialized.move_as_error_prefix("cannot serialize stack with get-method parameters : "));
|
||||||
|
}
|
||||||
|
|
||||||
|
auto query = ton::create_tl_object<ton::lite_api::liteServer_runSmcMethod>(
|
||||||
|
0x17, ton::create_tl_lite_block_id(block_id_), std::move(a), method_id, params_serialized.move_as_ok());
|
||||||
td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query,
|
td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query,
|
||||||
ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P));
|
ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P));
|
||||||
}
|
}
|
||||||
|
|
||||||
void HttpQueryRunMethod::got_account(td::BufferSlice data) {
|
void HttpQueryRunMethod::got_result(td::BufferSlice data) {
|
||||||
auto F = ton::fetch_tl_object<ton::lite_api::liteServer_accountState>(std::move(data), true);
|
auto F = ton::fetch_tl_object<ton::lite_api::liteServer_runMethodResult>(std::move(data), true);
|
||||||
if (F.is_error()) {
|
if (F.is_error()) {
|
||||||
abort_query(F.move_as_error());
|
return abort_query(F.move_as_error());
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
auto f = F.move_as_ok();
|
auto f = F.move_as_ok();
|
||||||
data_ = std::move(f->state_);
|
auto page = [&]() -> std::string {
|
||||||
proof_ = std::move(f->proof_);
|
HttpAnswer A{"account", prefix_};
|
||||||
shard_proof_ = std::move(f->shard_proof_);
|
A.set_account_id(addr_);
|
||||||
block_id_ = ton::create_block_id(f->id_);
|
A.set_block_id(ton::create_block_id(f->id_));
|
||||||
res_block_id_ = ton::create_block_id(f->shardblk_);
|
if (f->exit_code_ != 0) {
|
||||||
|
A.abort(PSTRING() << "VM terminated with error code " << f->exit_code_);
|
||||||
finish_query();
|
|
||||||
}
|
|
||||||
|
|
||||||
void HttpQueryRunMethod::finish_query() {
|
|
||||||
if (promise_) {
|
|
||||||
auto page = [&]() -> std::string {
|
|
||||||
HttpAnswer A{"account", prefix_};
|
|
||||||
A.set_account_id(addr_);
|
|
||||||
A.set_block_id(res_block_id_);
|
|
||||||
|
|
||||||
block::AccountState account_state;
|
|
||||||
account_state.blk = block_id_;
|
|
||||||
account_state.shard_blk = res_block_id_;
|
|
||||||
account_state.shard_proof = std::move(shard_proof_);
|
|
||||||
account_state.proof = std::move(proof_);
|
|
||||||
account_state.state = std::move(data_);
|
|
||||||
auto r_info = account_state.validate(block_id_, addr_);
|
|
||||||
if (r_info.is_error()) {
|
|
||||||
A.abort(r_info.move_as_error());
|
|
||||||
return A.finish();
|
|
||||||
}
|
|
||||||
auto info = r_info.move_as_ok();
|
|
||||||
if (info.root.is_null()) {
|
|
||||||
A.abort(PSTRING() << "account state of " << addr_ << " is empty (cannot run method `" << method_name_ << "`)");
|
|
||||||
return A.finish();
|
|
||||||
}
|
|
||||||
block::gen::Account::Record_account acc;
|
|
||||||
block::gen::AccountStorage::Record store;
|
|
||||||
block::CurrencyCollection balance;
|
|
||||||
if (!(tlb::unpack_cell(info.root, acc) && tlb::csr_unpack(acc.storage, store) &&
|
|
||||||
balance.validate_unpack(store.balance))) {
|
|
||||||
A.abort("error unpacking account state");
|
|
||||||
return A.finish();
|
|
||||||
}
|
|
||||||
int tag = block::gen::t_AccountState.get_tag(*store.state);
|
|
||||||
switch (tag) {
|
|
||||||
case block::gen::AccountState::account_uninit:
|
|
||||||
A.abort(PSTRING() << "account " << addr_ << " not initialized yet (cannot run any methods)");
|
|
||||||
return A.finish();
|
|
||||||
case block::gen::AccountState::account_frozen:
|
|
||||||
A.abort(PSTRING() << "account " << addr_ << " frozen (cannot run any methods)");
|
|
||||||
return A.finish();
|
|
||||||
}
|
|
||||||
|
|
||||||
CHECK(store.state.write().fetch_ulong(1) == 1); // account_init$1 _:StateInit = AccountState;
|
|
||||||
block::gen::StateInit::Record state_init;
|
|
||||||
CHECK(tlb::csr_unpack(store.state, state_init));
|
|
||||||
auto code = state_init.code->prefetch_ref();
|
|
||||||
auto data = state_init.data->prefetch_ref();
|
|
||||||
auto stack = td::make_ref<vm::Stack>(std::move(params_));
|
|
||||||
td::int64 method_id = (td::crc16(td::Slice{method_name_}) & 0xffff) | 0x10000;
|
|
||||||
stack.write().push_smallint(method_id);
|
|
||||||
long long gas_limit = vm::GasLimits::infty;
|
|
||||||
// OstreamLogger ostream_logger(ctx.error_stream);
|
|
||||||
// auto log = create_vm_log(ctx.error_stream ? &ostream_logger : nullptr);
|
|
||||||
vm::GasLimits gas{gas_limit};
|
|
||||||
LOG(DEBUG) << "creating VM";
|
|
||||||
vm::VmState vm{code, std::move(stack), gas, 1, data, vm::VmLog()};
|
|
||||||
vm.set_c7(prepare_vm_c7(info.gen_utime, info.gen_lt, acc.addr, balance)); // tuple with SmartContractInfo
|
|
||||||
// vm.incr_stack_trace(1); // enable stack dump after each step
|
|
||||||
int exit_code = ~vm.run();
|
|
||||||
if (exit_code != 0) {
|
|
||||||
A.abort(PSTRING() << "VM terminated with error code " << exit_code);
|
|
||||||
return A.finish();
|
|
||||||
}
|
|
||||||
stack = vm.get_stack_ref();
|
|
||||||
{
|
|
||||||
std::ostringstream os;
|
|
||||||
os << "result: ";
|
|
||||||
stack->dump(os, 3);
|
|
||||||
|
|
||||||
A << HttpAnswer::CodeBlock{os.str()};
|
|
||||||
}
|
|
||||||
|
|
||||||
return A.finish();
|
return A.finish();
|
||||||
}();
|
}
|
||||||
auto R = MHD_create_response_from_buffer(page.length(), const_cast<char *>(page.c_str()), MHD_RESPMEM_MUST_COPY);
|
|
||||||
MHD_add_response_header(R, "Content-Type", "text/html");
|
std::ostringstream os;
|
||||||
promise_.set_value(std::move(R));
|
os << "result: ";
|
||||||
}
|
if (f->result_.empty()) {
|
||||||
|
os << "<none>";
|
||||||
|
} else {
|
||||||
|
auto r_cell = vm::std_boc_deserialize(f->result_);
|
||||||
|
if (r_cell.is_error()) {
|
||||||
|
A.abort(PSTRING() << "cannot deserialize VM result boc: " << r_cell.move_as_error());
|
||||||
|
return A.finish();
|
||||||
|
}
|
||||||
|
auto cs = vm::load_cell_slice(r_cell.move_as_ok());
|
||||||
|
td::Ref<vm::Stack> stack;
|
||||||
|
if (!(vm::Stack::deserialize_to(cs, stack, 0) && cs.empty_ext())) {
|
||||||
|
A.abort("VM result boc cannot be deserialized");
|
||||||
|
return A.finish();
|
||||||
|
}
|
||||||
|
stack->dump(os, 3);
|
||||||
|
}
|
||||||
|
A << HttpAnswer::CodeBlock{os.str()};
|
||||||
|
return A.finish();
|
||||||
|
}();
|
||||||
|
auto R = MHD_create_response_from_buffer(page.length(), const_cast<char *>(page.c_str()), MHD_RESPMEM_MUST_COPY);
|
||||||
|
MHD_add_response_header(R, "Content-Type", "text/html");
|
||||||
|
promise_.set_value(std::move(R));
|
||||||
stop();
|
stop();
|
||||||
}
|
}
|
||||||
HttpQueryStatus::HttpQueryStatus(std::string prefix, td::Promise<MHD_Response *> promise)
|
HttpQueryStatus::HttpQueryStatus(std::string prefix, td::Promise<MHD_Response *> promise)
|
||||||
|
|
|
@ -311,22 +311,14 @@ class HttpQueryRunMethod : public HttpQueryCommon {
|
||||||
std::vector<vm::StackEntry> params, std::string prefix, td::Promise<MHD_Response *> promise);
|
std::vector<vm::StackEntry> params, std::string prefix, td::Promise<MHD_Response *> promise);
|
||||||
HttpQueryRunMethod(std::map<std::string, std::string> opts, std::string prefix, td::Promise<MHD_Response *> promise);
|
HttpQueryRunMethod(std::map<std::string, std::string> opts, std::string prefix, td::Promise<MHD_Response *> promise);
|
||||||
|
|
||||||
void finish_query();
|
|
||||||
|
|
||||||
void start_up_query() override;
|
void start_up_query() override;
|
||||||
void got_account(td::BufferSlice result);
|
void got_result(td::BufferSlice result);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
ton::BlockIdExt block_id_;
|
ton::BlockIdExt block_id_;
|
||||||
block::StdAddress addr_;
|
block::StdAddress addr_;
|
||||||
|
|
||||||
std::string method_name_;
|
std::string method_name_;
|
||||||
std::vector<vm::StackEntry> params_;
|
std::vector<vm::StackEntry> params_;
|
||||||
|
|
||||||
td::BufferSlice data_;
|
|
||||||
td::BufferSlice proof_;
|
|
||||||
td::BufferSlice shard_proof_;
|
|
||||||
ton::BlockIdExt res_block_id_;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
class HttpQueryStatus : public HttpQueryCommon {
|
class HttpQueryStatus : public HttpQueryCommon {
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
#include "td/utils/Status.h"
|
#include "td/utils/Status.h"
|
||||||
#include "td/utils/logging.h"
|
#include "td/utils/logging.h"
|
||||||
|
#include <functional>
|
||||||
namespace td {
|
namespace td {
|
||||||
class KeyValueReader {
|
class KeyValueReader {
|
||||||
public:
|
public:
|
||||||
|
@ -27,6 +28,9 @@ class KeyValueReader {
|
||||||
|
|
||||||
virtual Result<GetStatus> get(Slice key, std::string &value) = 0;
|
virtual Result<GetStatus> get(Slice key, std::string &value) = 0;
|
||||||
virtual Result<size_t> count(Slice prefix) = 0;
|
virtual Result<size_t> count(Slice prefix) = 0;
|
||||||
|
virtual Status for_each(std::function<Status(Slice, Slice)> f) {
|
||||||
|
return Status::Error("for_each is not supported");
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
class PrefixedKeyValueReader : public KeyValueReader {
|
class PrefixedKeyValueReader : public KeyValueReader {
|
||||||
|
|
|
@ -56,7 +56,7 @@ RocksDb::~RocksDb() {
|
||||||
}
|
}
|
||||||
|
|
||||||
RocksDb RocksDb::clone() const {
|
RocksDb RocksDb::clone() const {
|
||||||
return RocksDb{db_, statistics_};
|
return RocksDb{db_, options_};
|
||||||
}
|
}
|
||||||
|
|
||||||
Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
|
Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
|
||||||
|
@ -64,16 +64,16 @@ Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
|
||||||
{
|
{
|
||||||
rocksdb::Options db_options;
|
rocksdb::Options db_options;
|
||||||
|
|
||||||
static auto cache = rocksdb::NewLRUCache(1 << 30);
|
static auto default_cache = rocksdb::NewLRUCache(1 << 30);
|
||||||
|
if (options.block_cache == nullptr) {
|
||||||
|
options.block_cache = default_cache;
|
||||||
|
}
|
||||||
|
|
||||||
rocksdb::BlockBasedTableOptions table_options;
|
rocksdb::BlockBasedTableOptions table_options;
|
||||||
if (options.block_cache_size) {
|
table_options.block_cache = options.block_cache;
|
||||||
table_options.block_cache = rocksdb::NewLRUCache(options.block_cache_size.value());
|
|
||||||
} else {
|
|
||||||
table_options.block_cache = cache;
|
|
||||||
}
|
|
||||||
db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options));
|
db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options));
|
||||||
|
|
||||||
|
db_options.use_direct_reads = options.use_direct_reads;
|
||||||
db_options.manual_wal_flush = true;
|
db_options.manual_wal_flush = true;
|
||||||
db_options.create_if_missing = true;
|
db_options.create_if_missing = true;
|
||||||
db_options.max_background_compactions = 4;
|
db_options.max_background_compactions = 4;
|
||||||
|
@ -94,7 +94,7 @@ Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
|
||||||
// default column family
|
// default column family
|
||||||
delete handles[0];
|
delete handles[0];
|
||||||
}
|
}
|
||||||
return RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB>(db), std::move(options.statistics));
|
return RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB>(db), std::move(options));
|
||||||
}
|
}
|
||||||
|
|
||||||
std::shared_ptr<rocksdb::Statistics> RocksDb::create_statistics() {
|
std::shared_ptr<rocksdb::Statistics> RocksDb::create_statistics() {
|
||||||
|
@ -109,6 +109,10 @@ void RocksDb::reset_statistics(const std::shared_ptr<rocksdb::Statistics> statis
|
||||||
statistics->Reset();
|
statistics->Reset();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::shared_ptr<rocksdb::Cache> RocksDb::create_cache(size_t capacity) {
|
||||||
|
return rocksdb::NewLRUCache(capacity);
|
||||||
|
}
|
||||||
|
|
||||||
std::unique_ptr<KeyValueReader> RocksDb::snapshot() {
|
std::unique_ptr<KeyValueReader> RocksDb::snapshot() {
|
||||||
auto res = std::make_unique<RocksDb>(clone());
|
auto res = std::make_unique<RocksDb>(clone());
|
||||||
res->begin_snapshot().ensure();
|
res->begin_snapshot().ensure();
|
||||||
|
@ -120,7 +124,6 @@ std::string RocksDb::stats() const {
|
||||||
db_->GetProperty("rocksdb.stats", &out);
|
db_->GetProperty("rocksdb.stats", &out);
|
||||||
//db_->GetProperty("rocksdb.cur-size-all-mem-tables", &out);
|
//db_->GetProperty("rocksdb.cur-size-all-mem-tables", &out);
|
||||||
return out;
|
return out;
|
||||||
return statistics_->ToString();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Result<RocksDb::GetStatus> RocksDb::get(Slice key, std::string &value) {
|
Result<RocksDb::GetStatus> RocksDb::get(Slice key, std::string &value) {
|
||||||
|
@ -187,6 +190,28 @@ Result<size_t> RocksDb::count(Slice prefix) {
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Status RocksDb::for_each(std::function<Status(Slice, Slice)> f) {
|
||||||
|
rocksdb::ReadOptions options;
|
||||||
|
options.snapshot = snapshot_.get();
|
||||||
|
std::unique_ptr<rocksdb::Iterator> iterator;
|
||||||
|
if (snapshot_ || !transaction_) {
|
||||||
|
iterator.reset(db_->NewIterator(options));
|
||||||
|
} else {
|
||||||
|
iterator.reset(transaction_->GetIterator(options));
|
||||||
|
}
|
||||||
|
|
||||||
|
iterator->SeekToFirst();
|
||||||
|
for (; iterator->Valid(); iterator->Next()) {
|
||||||
|
auto key = from_rocksdb(iterator->key());
|
||||||
|
auto value = from_rocksdb(iterator->value());
|
||||||
|
TRY_STATUS(f(key, value));
|
||||||
|
}
|
||||||
|
if (!iterator->status().ok()) {
|
||||||
|
return from_rocksdb(iterator->status());
|
||||||
|
}
|
||||||
|
return Status::OK();
|
||||||
|
}
|
||||||
|
|
||||||
Status RocksDb::begin_write_batch() {
|
Status RocksDb::begin_write_batch() {
|
||||||
CHECK(!transaction_);
|
CHECK(!transaction_);
|
||||||
write_batch_ = std::make_unique<rocksdb::WriteBatch>();
|
write_batch_ = std::make_unique<rocksdb::WriteBatch>();
|
||||||
|
@ -243,7 +268,7 @@ Status RocksDb::end_snapshot() {
|
||||||
return td::Status::OK();
|
return td::Status::OK();
|
||||||
}
|
}
|
||||||
|
|
||||||
RocksDb::RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db, std::shared_ptr<rocksdb::Statistics> statistics)
|
RocksDb::RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db, RocksDbOptions options)
|
||||||
: db_(std::move(db)), statistics_(std::move(statistics)) {
|
: db_(std::move(db)), options_(options) {
|
||||||
}
|
}
|
||||||
} // namespace td
|
} // namespace td
|
||||||
|
|
|
@ -27,6 +27,7 @@
|
||||||
#include "td/utils/optional.h"
|
#include "td/utils/optional.h"
|
||||||
|
|
||||||
namespace rocksdb {
|
namespace rocksdb {
|
||||||
|
class Cache;
|
||||||
class OptimisticTransactionDB;
|
class OptimisticTransactionDB;
|
||||||
class Transaction;
|
class Transaction;
|
||||||
class WriteBatch;
|
class WriteBatch;
|
||||||
|
@ -38,7 +39,8 @@ namespace td {
|
||||||
|
|
||||||
struct RocksDbOptions {
|
struct RocksDbOptions {
|
||||||
std::shared_ptr<rocksdb::Statistics> statistics = nullptr;
|
std::shared_ptr<rocksdb::Statistics> statistics = nullptr;
|
||||||
optional<uint64> block_cache_size; // Default - one 1G cache for all RocksDb
|
std::shared_ptr<rocksdb::Cache> block_cache; // Default - one 1G cache for all RocksDb
|
||||||
|
bool use_direct_reads = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
class RocksDb : public KeyValue {
|
class RocksDb : public KeyValue {
|
||||||
|
@ -51,6 +53,7 @@ class RocksDb : public KeyValue {
|
||||||
Status set(Slice key, Slice value) override;
|
Status set(Slice key, Slice value) override;
|
||||||
Status erase(Slice key) override;
|
Status erase(Slice key) override;
|
||||||
Result<size_t> count(Slice prefix) override;
|
Result<size_t> count(Slice prefix) override;
|
||||||
|
Status for_each(std::function<Status(Slice, Slice)> f) override;
|
||||||
|
|
||||||
Status begin_write_batch() override;
|
Status begin_write_batch() override;
|
||||||
Status commit_write_batch() override;
|
Status commit_write_batch() override;
|
||||||
|
@ -71,6 +74,8 @@ class RocksDb : public KeyValue {
|
||||||
static std::string statistics_to_string(const std::shared_ptr<rocksdb::Statistics> statistics);
|
static std::string statistics_to_string(const std::shared_ptr<rocksdb::Statistics> statistics);
|
||||||
static void reset_statistics(const std::shared_ptr<rocksdb::Statistics> statistics);
|
static void reset_statistics(const std::shared_ptr<rocksdb::Statistics> statistics);
|
||||||
|
|
||||||
|
static std::shared_ptr<rocksdb::Cache> create_cache(size_t capacity);
|
||||||
|
|
||||||
RocksDb(RocksDb &&);
|
RocksDb(RocksDb &&);
|
||||||
RocksDb &operator=(RocksDb &&);
|
RocksDb &operator=(RocksDb &&);
|
||||||
~RocksDb();
|
~RocksDb();
|
||||||
|
@ -81,7 +86,7 @@ class RocksDb : public KeyValue {
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::shared_ptr<rocksdb::OptimisticTransactionDB> db_;
|
std::shared_ptr<rocksdb::OptimisticTransactionDB> db_;
|
||||||
std::shared_ptr<rocksdb::Statistics> statistics_;
|
RocksDbOptions options_;
|
||||||
|
|
||||||
std::unique_ptr<rocksdb::Transaction> transaction_;
|
std::unique_ptr<rocksdb::Transaction> transaction_;
|
||||||
std::unique_ptr<rocksdb::WriteBatch> write_batch_;
|
std::unique_ptr<rocksdb::WriteBatch> write_batch_;
|
||||||
|
@ -94,7 +99,6 @@ class RocksDb : public KeyValue {
|
||||||
};
|
};
|
||||||
std::unique_ptr<const rocksdb::Snapshot, UnreachableDeleter> snapshot_;
|
std::unique_ptr<const rocksdb::Snapshot, UnreachableDeleter> snapshot_;
|
||||||
|
|
||||||
explicit RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db,
|
explicit RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db, RocksDbOptions options);
|
||||||
std::shared_ptr<rocksdb::Statistics> statistics);
|
|
||||||
};
|
};
|
||||||
} // namespace td
|
} // namespace td
|
||||||
|
|
|
@ -1432,6 +1432,11 @@ td::Status ValidatorEngine::load_global_config() {
|
||||||
if (celldb_cache_size_) {
|
if (celldb_cache_size_) {
|
||||||
validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value());
|
validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value());
|
||||||
}
|
}
|
||||||
|
if (!celldb_cache_size_ || celldb_cache_size_.value() < (30ULL << 30)) {
|
||||||
|
celldb_direct_io_ = false;
|
||||||
|
}
|
||||||
|
validator_options_.write().set_celldb_direct_io(celldb_direct_io_);
|
||||||
|
validator_options_.write().set_celldb_preload_all(celldb_preload_all_);
|
||||||
if (catchain_max_block_delay_) {
|
if (catchain_max_block_delay_) {
|
||||||
validator_options_.write().set_catchain_max_block_delay(catchain_max_block_delay_.value());
|
validator_options_.write().set_catchain_max_block_delay(catchain_max_block_delay_.value());
|
||||||
}
|
}
|
||||||
|
@ -4285,8 +4290,7 @@ int main(int argc, char *argv[]) {
|
||||||
acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); });
|
acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); });
|
||||||
});
|
});
|
||||||
p.add_checked_option(
|
p.add_checked_option(
|
||||||
'\0', "celldb-cache-size",
|
'\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 50G)",
|
||||||
"block cache size for RocksDb in CellDb, in bytes (default: 1G cache shared by archive DB)",
|
|
||||||
[&](td::Slice s) -> td::Status {
|
[&](td::Slice s) -> td::Status {
|
||||||
TRY_RESULT(v, td::to_integer_safe<td::uint64>(s));
|
TRY_RESULT(v, td::to_integer_safe<td::uint64>(s));
|
||||||
if (v == 0) {
|
if (v == 0) {
|
||||||
|
@ -4295,6 +4299,13 @@ int main(int argc, char *argv[]) {
|
||||||
acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); });
|
acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); });
|
||||||
return td::Status::OK();
|
return td::Status::OK();
|
||||||
});
|
});
|
||||||
|
p.add_option(
|
||||||
|
'\0', "celldb-no-direct-io", "disable direct I/O mode for RocksDb in CellDb (forced when celldb cache is < 30G)",
|
||||||
|
[&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, false); }); });
|
||||||
|
p.add_option(
|
||||||
|
'\0', "celldb-no-preload-all",
|
||||||
|
"disable preloading all cells from CellDb on startup (enabled by default)",
|
||||||
|
[&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, false); }); });
|
||||||
p.add_checked_option(
|
p.add_checked_option(
|
||||||
'\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)",
|
'\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)",
|
||||||
[&](td::Slice s) -> td::Status {
|
[&](td::Slice s) -> td::Status {
|
||||||
|
|
|
@ -223,7 +223,9 @@ class ValidatorEngine : public td::actor::Actor {
|
||||||
double archive_preload_period_ = 0.0;
|
double archive_preload_period_ = 0.0;
|
||||||
bool disable_rocksdb_stats_ = false;
|
bool disable_rocksdb_stats_ = false;
|
||||||
bool nonfinal_ls_queries_enabled_ = false;
|
bool nonfinal_ls_queries_enabled_ = false;
|
||||||
td::optional<td::uint64> celldb_cache_size_;
|
td::optional<td::uint64> celldb_cache_size_ = 50LL << 30;
|
||||||
|
bool celldb_direct_io_ = true;
|
||||||
|
bool celldb_preload_all_ = true;
|
||||||
td::optional<double> catchain_max_block_delay_;
|
td::optional<double> catchain_max_block_delay_;
|
||||||
bool read_config_ = false;
|
bool read_config_ = false;
|
||||||
bool started_keyring_ = false;
|
bool started_keyring_ = false;
|
||||||
|
@ -303,6 +305,12 @@ class ValidatorEngine : public td::actor::Actor {
|
||||||
void set_celldb_cache_size(td::uint64 value) {
|
void set_celldb_cache_size(td::uint64 value) {
|
||||||
celldb_cache_size_ = value;
|
celldb_cache_size_ = value;
|
||||||
}
|
}
|
||||||
|
void set_celldb_direct_io(bool value) {
|
||||||
|
celldb_direct_io_ = value;
|
||||||
|
}
|
||||||
|
void set_celldb_preload_all(bool value) {
|
||||||
|
celldb_preload_all_ = value;
|
||||||
|
}
|
||||||
void set_catchain_max_block_delay(double value) {
|
void set_catchain_max_block_delay(double value) {
|
||||||
catchain_max_block_delay_ = value;
|
catchain_max_block_delay_ = value;
|
||||||
}
|
}
|
||||||
|
|
|
@ -91,9 +91,10 @@ void CellDbIn::start_up() {
|
||||||
td::RocksDbOptions db_options;
|
td::RocksDbOptions db_options;
|
||||||
db_options.statistics = statistics_;
|
db_options.statistics = statistics_;
|
||||||
if (opts_->get_celldb_cache_size()) {
|
if (opts_->get_celldb_cache_size()) {
|
||||||
db_options.block_cache_size = opts_->get_celldb_cache_size().value();
|
db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value());
|
||||||
LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(db_options.block_cache_size.value());
|
LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value());
|
||||||
}
|
}
|
||||||
|
db_options.use_direct_reads = opts_->get_celldb_direct_io();
|
||||||
cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_, std::move(db_options)).move_as_ok());
|
cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_, std::move(db_options)).move_as_ok());
|
||||||
|
|
||||||
|
|
||||||
|
@ -111,6 +112,27 @@ void CellDbIn::start_up() {
|
||||||
set_block(empty, std::move(e));
|
set_block(empty, std::move(e));
|
||||||
cell_db_->commit_write_batch().ensure();
|
cell_db_->commit_write_batch().ensure();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (opts_->get_celldb_preload_all()) {
|
||||||
|
// Iterate whole DB in a separate thread
|
||||||
|
delay_action([snapshot = cell_db_->snapshot()]() {
|
||||||
|
LOG(WARNING) << "CellDb: pre-loading all keys";
|
||||||
|
td::uint64 total = 0;
|
||||||
|
td::Timer timer;
|
||||||
|
auto S = snapshot->for_each([&](td::Slice, td::Slice) {
|
||||||
|
++total;
|
||||||
|
if (total % 1000000 == 0) {
|
||||||
|
LOG(INFO) << "CellDb: iterated " << total << " keys";
|
||||||
|
}
|
||||||
|
return td::Status::OK();
|
||||||
|
});
|
||||||
|
if (S.is_error()) {
|
||||||
|
LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error();
|
||||||
|
} else {
|
||||||
|
LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s";
|
||||||
|
}
|
||||||
|
}, td::Timestamp::now());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void CellDbIn::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
|
void CellDbIn::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
|
||||||
|
|
|
@ -130,6 +130,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
|
||||||
td::optional<td::uint64> get_celldb_cache_size() const override {
|
td::optional<td::uint64> get_celldb_cache_size() const override {
|
||||||
return celldb_cache_size_;
|
return celldb_cache_size_;
|
||||||
}
|
}
|
||||||
|
bool get_celldb_direct_io() const override {
|
||||||
|
return celldb_direct_io_;
|
||||||
|
}
|
||||||
|
bool get_celldb_preload_all() const override {
|
||||||
|
return celldb_preload_all_;
|
||||||
|
}
|
||||||
td::optional<double> get_catchain_max_block_delay() const override {
|
td::optional<double> get_catchain_max_block_delay() const override {
|
||||||
return catchain_max_block_delay_;
|
return catchain_max_block_delay_;
|
||||||
}
|
}
|
||||||
|
@ -207,6 +213,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
|
||||||
void set_celldb_cache_size(td::uint64 value) override {
|
void set_celldb_cache_size(td::uint64 value) override {
|
||||||
celldb_cache_size_ = value;
|
celldb_cache_size_ = value;
|
||||||
}
|
}
|
||||||
|
void set_celldb_direct_io(bool value) override {
|
||||||
|
celldb_direct_io_ = value;
|
||||||
|
}
|
||||||
|
void set_celldb_preload_all(bool value) override {
|
||||||
|
celldb_preload_all_ = value;
|
||||||
|
}
|
||||||
void set_catchain_max_block_delay(double value) override {
|
void set_catchain_max_block_delay(double value) override {
|
||||||
catchain_max_block_delay_ = value;
|
catchain_max_block_delay_ = value;
|
||||||
}
|
}
|
||||||
|
@ -259,6 +271,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
|
||||||
bool disable_rocksdb_stats_;
|
bool disable_rocksdb_stats_;
|
||||||
bool nonfinal_ls_queries_enabled_ = false;
|
bool nonfinal_ls_queries_enabled_ = false;
|
||||||
td::optional<td::uint64> celldb_cache_size_;
|
td::optional<td::uint64> celldb_cache_size_;
|
||||||
|
bool celldb_direct_io_ = false;
|
||||||
|
bool celldb_preload_all_ = false;
|
||||||
td::optional<double> catchain_max_block_delay_;
|
td::optional<double> catchain_max_block_delay_;
|
||||||
ValidatorMode validator_mode_ = validator_normal;
|
ValidatorMode validator_mode_ = validator_normal;
|
||||||
};
|
};
|
||||||
|
|
|
@ -87,6 +87,8 @@ struct ValidatorManagerOptions : public td::CntObject {
|
||||||
virtual bool get_disable_rocksdb_stats() const = 0;
|
virtual bool get_disable_rocksdb_stats() const = 0;
|
||||||
virtual bool nonfinal_ls_queries_enabled() const = 0;
|
virtual bool nonfinal_ls_queries_enabled() const = 0;
|
||||||
virtual td::optional<td::uint64> get_celldb_cache_size() const = 0;
|
virtual td::optional<td::uint64> get_celldb_cache_size() const = 0;
|
||||||
|
virtual bool get_celldb_direct_io() const = 0;
|
||||||
|
virtual bool get_celldb_preload_all() const = 0;
|
||||||
virtual td::optional<double> get_catchain_max_block_delay() const = 0;
|
virtual td::optional<double> get_catchain_max_block_delay() const = 0;
|
||||||
virtual ValidatorMode validator_mode() const = 0;
|
virtual ValidatorMode validator_mode() const = 0;
|
||||||
|
|
||||||
|
@ -113,6 +115,8 @@ struct ValidatorManagerOptions : public td::CntObject {
|
||||||
virtual void set_disable_rocksdb_stats(bool value) = 0;
|
virtual void set_disable_rocksdb_stats(bool value) = 0;
|
||||||
virtual void set_nonfinal_ls_queries_enabled(bool value) = 0;
|
virtual void set_nonfinal_ls_queries_enabled(bool value) = 0;
|
||||||
virtual void set_celldb_cache_size(td::uint64 value) = 0;
|
virtual void set_celldb_cache_size(td::uint64 value) = 0;
|
||||||
|
virtual void set_celldb_direct_io(bool value) = 0;
|
||||||
|
virtual void set_celldb_preload_all(bool value) = 0;
|
||||||
virtual void set_catchain_max_block_delay(double value) = 0;
|
virtual void set_catchain_max_block_delay(double value) = 0;
|
||||||
virtual void set_validator_mode(ValidatorMode value) = 0;
|
virtual void set_validator_mode(ValidatorMode value) = 0;
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue