mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
celldb in-memory mode, stats for actors, perf counters, minor fix in rldp2 (#1164)
* getactorstats query for validator-engine-console * celldb in-memory mode (--celldb-in-memory option) * rldp2: bugfix - do not estimate speed while nothing is sent * add simple ed25519 benchmark * fix compilation errors of different platforms and move to c++20 * fix some warnings * turn on TON_USE_ABSEIL for glibc 2.27 nix build --------- Co-authored-by: birydrad <>
This commit is contained in:
birydrad
GitHub
parent
5f51d3d04f
commit
72020c04c4
100 changed files with 3407 additions and 359 deletions
|
|
@ -32,11 +32,11 @@ std::string PackageId::path() const {
|
|||
return "/files/packages/";
|
||||
} else if (key) {
|
||||
char s[24];
|
||||
sprintf(s, "key%03d", id / 1000000);
|
||||
snprintf(s, sizeof(s), "key%03d", id / 1000000);
|
||||
return PSTRING() << "/archive/packages/" << s << "/";
|
||||
} else {
|
||||
char s[20];
|
||||
sprintf(s, "arch%04d", id / 100000);
|
||||
snprintf(s, sizeof(s), "arch%04d", id / 100000);
|
||||
return PSTRING() << "/archive/packages/" << s << "/";
|
||||
}
|
||||
}
|
||||
|
|
@ -46,11 +46,11 @@ std::string PackageId::name() const {
|
|||
return PSTRING() << "temp.archive." << id;
|
||||
} else if (key) {
|
||||
char s[20];
|
||||
sprintf(s, "%06d", id);
|
||||
snprintf(s, sizeof(s), "%06d", id);
|
||||
return PSTRING() << "key.archive." << s;
|
||||
} else {
|
||||
char s[10];
|
||||
sprintf(s, "%05d", id);
|
||||
snprintf(s, sizeof(s), "%05d", id);
|
||||
return PSTRING() << "archive." << s;
|
||||
}
|
||||
}
|
||||
|
|
@ -342,19 +342,19 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t
|
|||
void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data,
|
||||
td::Promise<td::Unit> promise) {
|
||||
auto create_writer = [&](std::string path, td::Promise<std::string> P) {
|
||||
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/",
|
||||
std::move(path), std::move(data), std::move(P))
|
||||
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/", std::move(path), std::move(data),
|
||||
std::move(P))
|
||||
.release();
|
||||
};
|
||||
add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer));
|
||||
}
|
||||
|
||||
void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
|
||||
std::function<td::Status(td::FileFd&)> write_state,
|
||||
std::function<td::Status(td::FileFd &)> write_state,
|
||||
td::Promise<td::Unit> promise) {
|
||||
auto create_writer = [&](std::string path, td::Promise<std::string> P) {
|
||||
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/",
|
||||
std::move(path), std::move(write_state), std::move(P))
|
||||
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/", std::move(path),
|
||||
std::move(write_state), std::move(P))
|
||||
.release();
|
||||
};
|
||||
add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer));
|
||||
|
|
@ -624,8 +624,8 @@ void ArchiveManager::load_package(PackageId id) {
|
|||
}
|
||||
}
|
||||
|
||||
desc.file =
|
||||
td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_);
|
||||
desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_,
|
||||
archive_lru_.get(), statistics_);
|
||||
|
||||
m.emplace(id, std::move(desc));
|
||||
update_permanent_slices();
|
||||
|
|
@ -659,8 +659,8 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull
|
|||
FileDescription new_desc{id, false};
|
||||
td::mkdir(db_root_ + id.path()).ensure();
|
||||
std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
|
||||
new_desc.file =
|
||||
td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_);
|
||||
new_desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_,
|
||||
archive_lru_.get(), statistics_);
|
||||
const FileDescription &desc = f.emplace(id, std::move(new_desc));
|
||||
if (!id.temp) {
|
||||
update_desc(f, desc, shard, seqno, ts, lt);
|
||||
|
|
@ -940,7 +940,8 @@ void ArchiveManager::start_up() {
|
|||
void ArchiveManager::alarm() {
|
||||
alarm_timestamp() = td::Timestamp::in(60.0);
|
||||
auto stats = statistics_.to_string_and_reset();
|
||||
auto to_file_r = td::FileFd::open(db_root_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644);
|
||||
auto to_file_r =
|
||||
td::FileFd::open(db_root_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644);
|
||||
if (to_file_r.is_error()) {
|
||||
LOG(ERROR) << "Failed to open db_stats.txt: " << to_file_r.move_as_error();
|
||||
return;
|
||||
|
|
@ -1034,7 +1035,7 @@ void ArchiveManager::persistent_state_gc(std::pair<BlockSeqno, FileHash> last) {
|
|||
}
|
||||
if (res != 0) {
|
||||
delay_action([key, SelfId = actor_id(
|
||||
this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
td::Timestamp::in(1.0));
|
||||
return;
|
||||
}
|
||||
|
|
@ -1051,7 +1052,7 @@ void ArchiveManager::persistent_state_gc(std::pair<BlockSeqno, FileHash> last) {
|
|||
}
|
||||
if (!allow_delete) {
|
||||
delay_action([key, SelfId = actor_id(
|
||||
this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
td::Timestamp::in(1.0));
|
||||
return;
|
||||
}
|
||||
|
|
@ -1082,9 +1083,9 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pai
|
|||
td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore();
|
||||
perm_states_.erase(it);
|
||||
}
|
||||
delay_action([key, SelfId = actor_id(
|
||||
this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
td::Timestamp::in(1.0));
|
||||
delay_action(
|
||||
[key, SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
td::Timestamp::in(1.0));
|
||||
}
|
||||
|
||||
PackageId ArchiveManager::get_temp_package_id() const {
|
||||
|
|
|
|||
|
|
@ -17,10 +17,12 @@
|
|||
Copyright 2017-2020 Telegram Systems LLP
|
||||
*/
|
||||
#include "celldb.hpp"
|
||||
|
||||
#include "files-async.hpp"
|
||||
#include "rootdb.hpp"
|
||||
|
||||
#include "td/db/RocksDb.h"
|
||||
#include "td/utils/filesystem.h"
|
||||
#include "rocksdb/utilities/optimistic_transaction_db.h"
|
||||
|
||||
#include "ton/ton-tl.hpp"
|
||||
#include "ton/ton-io.hpp"
|
||||
|
|
@ -29,7 +31,6 @@
|
|||
namespace ton {
|
||||
|
||||
namespace validator {
|
||||
|
||||
class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor {
|
||||
public:
|
||||
explicit CellDbAsyncExecutor(td::actor::ActorId<CellDbBase> cell_db) : cell_db_(std::move(cell_db)) {
|
||||
|
|
@ -38,11 +39,13 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor {
|
|||
void execute_async(std::function<void()> f) override {
|
||||
class Runner : public td::actor::Actor {
|
||||
public:
|
||||
explicit Runner(std::function<void()> f) : f_(std::move(f)) {}
|
||||
explicit Runner(std::function<void()> f) : f_(std::move(f)) {
|
||||
}
|
||||
void start_up() override {
|
||||
f_();
|
||||
stop();
|
||||
}
|
||||
|
||||
private:
|
||||
std::function<void()> f_;
|
||||
};
|
||||
|
|
@ -52,6 +55,7 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor {
|
|||
void execute_sync(std::function<void()> f) override {
|
||||
td::actor::send_closure(cell_db_, &CellDbBase::execute_sync, std::move(f));
|
||||
}
|
||||
|
||||
private:
|
||||
td::actor::ActorId<CellDbBase> cell_db_;
|
||||
};
|
||||
|
|
@ -97,13 +101,30 @@ void CellDbIn::start_up() {
|
|||
LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value());
|
||||
}
|
||||
db_options.use_direct_reads = opts_->get_celldb_direct_io();
|
||||
cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_, std::move(db_options)).move_as_ok());
|
||||
|
||||
if (opts_->get_celldb_in_memory()) {
|
||||
td::RocksDbOptions read_db_options;
|
||||
read_db_options.use_direct_reads = true;
|
||||
read_db_options.no_block_cache = true;
|
||||
read_db_options.block_cache = {};
|
||||
LOG(WARNING) << "Loading all cells in memory (because of --celldb-in-memory)";
|
||||
td::Timer timer;
|
||||
auto read_cell_db =
|
||||
std::make_shared<td::RocksDb>(td::RocksDb::open(path_, std::move(read_db_options)).move_as_ok());
|
||||
boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), {});
|
||||
in_memory_load_time_ = timer.elapsed();
|
||||
td::actor::send_closure(parent_, &CellDb::set_in_memory_boc, boc_);
|
||||
}
|
||||
|
||||
boc_ = vm::DynamicBagOfCellsDb::create();
|
||||
boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth());
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
auto rocks_db = std::make_shared<td::RocksDb>(td::RocksDb::open(path_, std::move(db_options)).move_as_ok());
|
||||
rocks_db_ = rocks_db->raw_db();
|
||||
cell_db_ = std::move(rocks_db);
|
||||
if (!opts_->get_celldb_in_memory()) {
|
||||
boc_ = vm::DynamicBagOfCellsDb::create();
|
||||
boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth());
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
}
|
||||
|
||||
alarm_timestamp() = td::Timestamp::in(10.0);
|
||||
|
||||
|
|
@ -117,31 +138,39 @@ void CellDbIn::start_up() {
|
|||
|
||||
if (opts_->get_celldb_preload_all()) {
|
||||
// Iterate whole DB in a separate thread
|
||||
delay_action([snapshot = cell_db_->snapshot()]() {
|
||||
LOG(WARNING) << "CellDb: pre-loading all keys";
|
||||
td::uint64 total = 0;
|
||||
td::Timer timer;
|
||||
auto S = snapshot->for_each([&](td::Slice, td::Slice) {
|
||||
++total;
|
||||
if (total % 1000000 == 0) {
|
||||
LOG(INFO) << "CellDb: iterated " << total << " keys";
|
||||
}
|
||||
return td::Status::OK();
|
||||
});
|
||||
if (S.is_error()) {
|
||||
LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error();
|
||||
} else {
|
||||
LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s";
|
||||
}
|
||||
}, td::Timestamp::now());
|
||||
delay_action(
|
||||
[snapshot = cell_db_->snapshot()]() {
|
||||
LOG(WARNING) << "CellDb: pre-loading all keys";
|
||||
td::uint64 total = 0;
|
||||
td::Timer timer;
|
||||
auto S = snapshot->for_each([&](td::Slice, td::Slice) {
|
||||
++total;
|
||||
if (total % 1000000 == 0) {
|
||||
LOG(INFO) << "CellDb: iterated " << total << " keys";
|
||||
}
|
||||
return td::Status::OK();
|
||||
});
|
||||
if (S.is_error()) {
|
||||
LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error();
|
||||
} else {
|
||||
LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s";
|
||||
}
|
||||
},
|
||||
td::Timestamp::now());
|
||||
}
|
||||
}
|
||||
|
||||
void CellDbIn::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
|
||||
if (opts_->get_celldb_in_memory()) {
|
||||
auto result = boc_->load_root(hash.as_slice());
|
||||
async_apply("load_cell_result", std::move(promise), std::move(result));
|
||||
return;
|
||||
}
|
||||
boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise));
|
||||
}
|
||||
|
||||
void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise) {
|
||||
TD_PERF_COUNTER(celldb_store_cell);
|
||||
td::PerfWarningTimer timer{"storecell", 0.1};
|
||||
auto key_hash = get_key_hash(block_id);
|
||||
auto R = get_block(key_hash);
|
||||
|
|
@ -181,8 +210,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promi
|
|||
set_block(key_hash, std::move(D));
|
||||
cell_db_->commit_write_batch().ensure();
|
||||
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
if (!opts_->get_celldb_in_memory()) {
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
}
|
||||
|
||||
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
|
||||
if (!opts_->get_disable_rocksdb_stats()) {
|
||||
|
|
@ -199,12 +230,50 @@ void CellDbIn::get_last_deleted_mc_state(td::Promise<BlockSeqno> promise) {
|
|||
promise.set_result(last_deleted_mc_state_);
|
||||
}
|
||||
|
||||
std::vector<std::pair<std::string, std::string>> CellDbIn::prepare_stats() {
|
||||
TD_PERF_COUNTER(celldb_prepare_stats);
|
||||
auto r_boc_stats = boc_->get_stats();
|
||||
if (r_boc_stats.is_ok()) {
|
||||
cell_db_statistics_.boc_stats_ = r_boc_stats.move_as_ok();
|
||||
}
|
||||
cell_db_statistics_.in_memory_load_time_ = in_memory_load_time_;
|
||||
auto stats = cell_db_statistics_.prepare_stats();
|
||||
auto add_stat = [&](const auto& key, const auto& value) { stats.emplace_back(key, PSTRING() << value); };
|
||||
|
||||
add_stat("started", "true");
|
||||
auto r_mem_stat = td::mem_stat();
|
||||
auto r_total_mem_stat = td::get_total_mem_stat();
|
||||
td::uint64 celldb_size = 0;
|
||||
bool ok_celldb_size = rocks_db_->GetIntProperty("rocksdb.total-sst-files-size", &celldb_size);
|
||||
if (celldb_size > 0 && r_mem_stat.is_ok() && r_total_mem_stat.is_ok() && ok_celldb_size) {
|
||||
auto mem_stat = r_mem_stat.move_as_ok();
|
||||
auto total_mem_stat = r_total_mem_stat.move_as_ok();
|
||||
add_stat("rss", td::format::as_size(mem_stat.resident_size_));
|
||||
add_stat("available_ram", td::format::as_size(total_mem_stat.available_ram));
|
||||
add_stat("total_ram", td::format::as_size(total_mem_stat.total_ram));
|
||||
add_stat("actual_ram_to_celldb_ratio", double(total_mem_stat.available_ram) / double(celldb_size));
|
||||
add_stat("if_restarted_ram_to_celldb_ratio",
|
||||
double(total_mem_stat.available_ram + mem_stat.resident_size_ - 10 * (td::uint64(1) << 30)) /
|
||||
double(celldb_size));
|
||||
add_stat("max_possible_ram_to_celldb_ratio", double(total_mem_stat.total_ram) / double(celldb_size));
|
||||
}
|
||||
|
||||
return stats;
|
||||
// do not clear statistics, it is needed for flush_db_stats
|
||||
}
|
||||
void CellDbIn::flush_db_stats() {
|
||||
if (opts_->get_disable_rocksdb_stats()) {
|
||||
return;
|
||||
}
|
||||
auto stats = td::RocksDb::statistics_to_string(statistics_) + snapshot_statistics_->to_string() +
|
||||
cell_db_statistics_.to_string();
|
||||
|
||||
auto celldb_stats = prepare_stats();
|
||||
td::StringBuilder ss;
|
||||
for (auto& [key, value] : celldb_stats) {
|
||||
ss << "ton.celldb." << key << " " << value << "\n";
|
||||
}
|
||||
|
||||
auto stats =
|
||||
td::RocksDb::statistics_to_string(statistics_) + snapshot_statistics_->to_string() + ss.as_cslice().str();
|
||||
auto to_file_r =
|
||||
td::FileFd::open(path_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644);
|
||||
if (to_file_r.is_error()) {
|
||||
|
|
@ -284,8 +353,11 @@ void CellDbIn::gc_cont(BlockHandle handle) {
|
|||
}
|
||||
|
||||
void CellDbIn::gc_cont2(BlockHandle handle) {
|
||||
TD_PERF_COUNTER(celldb_gc_cell);
|
||||
td::PerfWarningTimer timer{"gccell", 0.1};
|
||||
td::PerfWarningTimer timer_all{"gccell_all", 0.05};
|
||||
|
||||
td::PerfWarningTimer timer_get_keys{"gccell_get_keys", 0.05};
|
||||
auto key_hash = get_key_hash(handle->id());
|
||||
auto FR = get_block(key_hash);
|
||||
FR.ensure();
|
||||
|
|
@ -304,22 +376,41 @@ void CellDbIn::gc_cont2(BlockHandle handle) {
|
|||
P.prev = P.next;
|
||||
N.next = N.prev;
|
||||
}
|
||||
timer_get_keys.reset();
|
||||
|
||||
td::PerfWarningTimer timer_boc{"gccell_boc", 0.05};
|
||||
auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok();
|
||||
|
||||
boc_->dec(cell);
|
||||
boc_->prepare_commit().ensure();
|
||||
vm::CellStorer stor{*cell_db_};
|
||||
timer_boc.reset();
|
||||
|
||||
td::PerfWarningTimer timer_write_batch{"gccell_write_batch", 0.05};
|
||||
cell_db_->begin_write_batch().ensure();
|
||||
boc_->commit(stor).ensure();
|
||||
|
||||
cell_db_->erase(get_key(key_hash)).ensure();
|
||||
set_block(F.prev, std::move(P));
|
||||
set_block(F.next, std::move(N));
|
||||
cell_db_->commit_write_batch().ensure();
|
||||
alarm_timestamp() = td::Timestamp::now();
|
||||
timer_write_batch.reset();
|
||||
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
td::PerfWarningTimer timer_free_cells{"gccell_free_cells", 0.05};
|
||||
auto before = td::ref_get_delete_count();
|
||||
cell = {};
|
||||
auto after = td::ref_get_delete_count();
|
||||
if (timer_free_cells.elapsed() > 0.04) {
|
||||
LOG(ERROR) << "deleted " << after - before << " cells";
|
||||
}
|
||||
timer_free_cells.reset();
|
||||
|
||||
td::PerfWarningTimer timer_finish{"gccell_finish", 0.05};
|
||||
if (!opts_->get_celldb_in_memory()) {
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
}
|
||||
|
||||
DCHECK(get_block(key_hash).is_error());
|
||||
if (!opts_->get_disable_rocksdb_stats()) {
|
||||
|
|
@ -329,6 +420,8 @@ void CellDbIn::gc_cont2(BlockHandle handle) {
|
|||
last_deleted_mc_state_ = handle->id().seqno();
|
||||
}
|
||||
LOG(DEBUG) << "Deleted state " << handle->id().to_str();
|
||||
timer_finish.reset();
|
||||
timer_all.reset();
|
||||
}
|
||||
|
||||
void CellDbIn::skip_gc() {
|
||||
|
|
@ -401,7 +494,7 @@ void CellDbIn::migrate_cells() {
|
|||
boc_->set_loader(std::make_unique<vm::CellLoader>(*loader)).ensure();
|
||||
cell_db_->begin_write_batch().ensure();
|
||||
td::uint32 checked = 0, migrated = 0;
|
||||
for (auto it = cells_to_migrate_.begin(); it != cells_to_migrate_.end() && checked < 128; ) {
|
||||
for (auto it = cells_to_migrate_.begin(); it != cells_to_migrate_.end() && checked < 128;) {
|
||||
++checked;
|
||||
td::Bits256 hash = *it;
|
||||
it = cells_to_migrate_.erase(it);
|
||||
|
|
@ -438,7 +531,32 @@ void CellDbIn::migrate_cells() {
|
|||
}
|
||||
}
|
||||
|
||||
void CellDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
|
||||
promise.set_value(decltype(prepared_stats_)(prepared_stats_));
|
||||
}
|
||||
|
||||
void CellDb::update_stats(td::Result<std::vector<std::pair<std::string, std::string>>> r_stats) {
|
||||
if (r_stats.is_error()) {
|
||||
LOG(ERROR) << "error updating stats: " << r_stats.error();
|
||||
} else {
|
||||
prepared_stats_ = r_stats.move_as_ok();
|
||||
}
|
||||
alarm_timestamp() = td::Timestamp::in(2.0);
|
||||
}
|
||||
|
||||
void CellDb::alarm() {
|
||||
send_closure(cell_db_, &CellDbIn::prepare_stats, td::promise_send_closure(actor_id(this), &CellDb::update_stats));
|
||||
}
|
||||
|
||||
void CellDb::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
|
||||
if (in_memory_boc_) {
|
||||
auto result = in_memory_boc_->load_root_thread_safe(hash.as_slice());
|
||||
if (result.is_ok()) {
|
||||
return async_apply("load_cell_result", std::move(promise), std::move(result));
|
||||
} else {
|
||||
LOG(ERROR) << "load_root_thread_safe failed - this is suspicious";
|
||||
}
|
||||
}
|
||||
if (!started_) {
|
||||
td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise));
|
||||
} else {
|
||||
|
|
@ -496,12 +614,24 @@ td::BufferSlice CellDbIn::DbEntry::release() {
|
|||
return create_serialize_tl_object<ton_api::db_celldb_value>(create_tl_block_id(block_id), prev, next, root_hash);
|
||||
}
|
||||
|
||||
std::string CellDbIn::CellDbStatistics::to_string() {
|
||||
td::StringBuilder ss;
|
||||
ss << "ton.celldb.store_cell.micros " << store_cell_time_.to_string() << "\n";
|
||||
ss << "ton.celldb.gc_cell.micros " << gc_cell_time_.to_string() << "\n";
|
||||
ss << "ton.celldb.total_time.micros : " << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6 << "\n";
|
||||
return ss.as_cslice().str();
|
||||
std::vector<std::pair<std::string, std::string>> CellDbIn::CellDbStatistics::prepare_stats() {
|
||||
std::vector<std::pair<std::string, std::string>> stats;
|
||||
stats.emplace_back("store_cell.micros", PSTRING() << store_cell_time_.to_string());
|
||||
stats.emplace_back("gc_cell.micros", PSTRING() << gc_cell_time_.to_string());
|
||||
stats.emplace_back("total_time.micros", PSTRING() << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6);
|
||||
stats.emplace_back("in_memory", PSTRING() << bool(in_memory_load_time_));
|
||||
if (in_memory_load_time_) {
|
||||
stats.emplace_back("in_memory_load_time", PSTRING() << in_memory_load_time_.value());
|
||||
}
|
||||
if (boc_stats_) {
|
||||
stats.emplace_back("cells_count", PSTRING() << boc_stats_->cells_total_count);
|
||||
stats.emplace_back("cells_size", PSTRING() << boc_stats_->cells_total_size);
|
||||
stats.emplace_back("roots_count", PSTRING() << boc_stats_->roots_total_count);
|
||||
for (auto& [key, value] : boc_stats_->custom_stats) {
|
||||
stats.emplace_back(key, value);
|
||||
}
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
} // namespace validator
|
||||
|
|
|
|||
|
|
@ -29,9 +29,12 @@
|
|||
#include "db-utils.h"
|
||||
#include "td/db/RocksDb.h"
|
||||
|
||||
#include <optional>
|
||||
|
||||
namespace rocksdb {
|
||||
class Statistics;
|
||||
}
|
||||
class DB;
|
||||
} // namespace rocksdb
|
||||
|
||||
namespace ton {
|
||||
|
||||
|
|
@ -58,6 +61,7 @@ class CellDbIn : public CellDbBase {
|
|||
public:
|
||||
using KeyHash = td::Bits256;
|
||||
|
||||
std::vector<std::pair<std::string, std::string>> prepare_stats();
|
||||
void load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise);
|
||||
void store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise);
|
||||
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise);
|
||||
|
|
@ -111,13 +115,15 @@ class CellDbIn : public CellDbBase {
|
|||
std::string path_;
|
||||
td::Ref<ValidatorManagerOptions> opts_;
|
||||
|
||||
std::unique_ptr<vm::DynamicBagOfCellsDb> boc_;
|
||||
std::shared_ptr<vm::DynamicBagOfCellsDb> boc_;
|
||||
std::shared_ptr<vm::KeyValue> cell_db_;
|
||||
std::shared_ptr<rocksdb::DB> rocks_db_;
|
||||
|
||||
std::function<void(const vm::CellLoader::LoadResult&)> on_load_callback_;
|
||||
std::set<td::Bits256> cells_to_migrate_;
|
||||
td::Timestamp migrate_after_ = td::Timestamp::never();
|
||||
bool migration_active_ = false;
|
||||
std::optional<double> in_memory_load_time_;
|
||||
|
||||
struct MigrationStats {
|
||||
td::Timer start_;
|
||||
|
|
@ -133,8 +139,10 @@ class CellDbIn : public CellDbBase {
|
|||
PercentileStats store_cell_time_;
|
||||
PercentileStats gc_cell_time_;
|
||||
td::Timestamp stats_start_time_ = td::Timestamp::now();
|
||||
std::optional<double> in_memory_load_time_;
|
||||
std::optional<vm::DynamicBagOfCellsDb::Stats> boc_stats_;
|
||||
|
||||
std::string to_string();
|
||||
std::vector<std::pair<std::string, std::string>> prepare_stats();
|
||||
void clear() {
|
||||
*this = CellDbStatistics{};
|
||||
}
|
||||
|
|
@ -162,12 +170,25 @@ class CellDbIn : public CellDbBase {
|
|||
|
||||
class CellDb : public CellDbBase {
|
||||
public:
|
||||
void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise);
|
||||
void load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise);
|
||||
void store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise);
|
||||
void update_snapshot(std::unique_ptr<td::KeyValueReader> snapshot) {
|
||||
CHECK(!opts_->get_celldb_in_memory());
|
||||
if (!started_) {
|
||||
alarm();
|
||||
}
|
||||
started_ = true;
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(std::move(snapshot), on_load_callback_)).ensure();
|
||||
}
|
||||
void set_in_memory_boc(std::shared_ptr<const vm::DynamicBagOfCellsDb> in_memory_boc) {
|
||||
CHECK(opts_->get_celldb_in_memory());
|
||||
if (!started_) {
|
||||
alarm();
|
||||
}
|
||||
started_ = true;
|
||||
in_memory_boc_ = std::move(in_memory_boc);
|
||||
}
|
||||
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise);
|
||||
void get_last_deleted_mc_state(td::Promise<BlockSeqno> promise);
|
||||
|
||||
|
|
@ -185,9 +206,14 @@ class CellDb : public CellDbBase {
|
|||
td::actor::ActorOwn<CellDbIn> cell_db_;
|
||||
|
||||
std::unique_ptr<vm::DynamicBagOfCellsDb> boc_;
|
||||
std::shared_ptr<const vm::DynamicBagOfCellsDb> in_memory_boc_;
|
||||
bool started_ = false;
|
||||
std::vector<std::pair<std::string, std::string>> prepared_stats_{{"started", "false"}};
|
||||
|
||||
std::function<void(const vm::CellLoader::LoadResult&)> on_load_callback_;
|
||||
|
||||
void update_stats(td::Result<std::vector<std::pair<std::string, std::string>>> stats);
|
||||
void alarm() override;
|
||||
};
|
||||
|
||||
} // namespace validator
|
||||
|
|
|
|||
|
|
@ -79,14 +79,14 @@ std::string Block::filename() const {
|
|||
|
||||
std::string Block::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "block_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string BlockShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "block_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
@ -116,14 +116,14 @@ std::string PersistentState::filename() const {
|
|||
|
||||
std::string PersistentState::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "state_" << masterchain_block_id.seqno() << "_" << block_id.id.workchain << "_" << s << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string PersistentStateShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(shard_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(shard_id.shard));
|
||||
return PSTRING() << "state_" << masterchain_seqno << "_" << shard_id.workchain << "_" << s << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
@ -137,14 +137,14 @@ std::string Proof::filename() const {
|
|||
|
||||
std::string Proof::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "proof_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string ProofShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "proof_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
@ -158,14 +158,14 @@ std::string ProofLink::filename() const {
|
|||
|
||||
std::string ProofLink::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "prooflink_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string ProofLinkShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "prooflink_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
@ -179,14 +179,14 @@ std::string Signatures::filename() const {
|
|||
|
||||
std::string Signatures::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "signatures_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string SignaturesShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "signatures_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
|
@ -202,14 +202,14 @@ std::string Candidate::filename() const {
|
|||
|
||||
std::string Candidate::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "candidate_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string CandidateShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "candidate_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
@ -223,14 +223,14 @@ std::string CandidateRef::filename() const {
|
|||
|
||||
std::string CandidateRef::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "candidateref_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string CandidateRefShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "candidateref_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
|
@ -245,14 +245,14 @@ std::string BlockInfo::filename() const {
|
|||
|
||||
std::string BlockInfo::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "info_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string BlockInfoShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "info_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -439,6 +439,7 @@ void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise<bool> promise) {
|
|||
|
||||
void RootDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
|
||||
auto merger = StatsMerger::create(std::move(promise));
|
||||
td::actor::send_closure(cell_db_, &CellDb::prepare_stats, merger.make_promise("celldb."));
|
||||
}
|
||||
|
||||
void RootDb::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue