mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
Merge branch 'testnet' into accelerator
This commit is contained in:
SpyCheese
commit
ed6788e579
113 changed files with 3450 additions and 420 deletions
|
|
@ -205,14 +205,10 @@ target_include_directories(full-node PUBLIC
|
|||
${OPENSSL_INCLUDE_DIR}
|
||||
)
|
||||
|
||||
target_link_libraries(validator PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec
|
||||
overlay catchain validatorsession ton_crypto ton_block ton_db)
|
||||
target_link_libraries(validator PRIVATE tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_db)
|
||||
|
||||
target_link_libraries(validator-disk PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec
|
||||
overlay catchain validatorsession ton_crypto ton_block ton_db)
|
||||
target_link_libraries(validator-disk PRIVATE tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_db)
|
||||
|
||||
target_link_libraries(validator-hardfork PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec
|
||||
overlay catchain validatorsession ton_crypto ton_block ton_db)
|
||||
target_link_libraries(validator-hardfork PRIVATE tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_db)
|
||||
|
||||
target_link_libraries(full-node PRIVATE tdutils tdactor adnl rldp rldp2 tl_api dht tdfec
|
||||
overlay catchain validatorsession ton_crypto ton_block ton_db)
|
||||
target_link_libraries(full-node PRIVATE tdactor adnl rldp rldp2 tl_api dht tdfec overlay catchain validatorsession ton_db)
|
||||
|
|
|
|||
|
|
@ -32,11 +32,11 @@ std::string PackageId::path() const {
|
|||
return "/files/packages/";
|
||||
} else if (key) {
|
||||
char s[24];
|
||||
sprintf(s, "key%03d", id / 1000000);
|
||||
snprintf(s, sizeof(s), "key%03d", id / 1000000);
|
||||
return PSTRING() << "/archive/packages/" << s << "/";
|
||||
} else {
|
||||
char s[20];
|
||||
sprintf(s, "arch%04d", id / 100000);
|
||||
snprintf(s, sizeof(s), "arch%04d", id / 100000);
|
||||
return PSTRING() << "/archive/packages/" << s << "/";
|
||||
}
|
||||
}
|
||||
|
|
@ -46,11 +46,11 @@ std::string PackageId::name() const {
|
|||
return PSTRING() << "temp.archive." << id;
|
||||
} else if (key) {
|
||||
char s[20];
|
||||
sprintf(s, "%06d", id);
|
||||
snprintf(s, sizeof(s), "%06d", id);
|
||||
return PSTRING() << "key.archive." << s;
|
||||
} else {
|
||||
char s[10];
|
||||
sprintf(s, "%05d", id);
|
||||
snprintf(s, sizeof(s), "%05d", id);
|
||||
return PSTRING() << "archive." << s;
|
||||
}
|
||||
}
|
||||
|
|
@ -342,19 +342,19 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t
|
|||
void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data,
|
||||
td::Promise<td::Unit> promise) {
|
||||
auto create_writer = [&](std::string path, td::Promise<std::string> P) {
|
||||
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/",
|
||||
std::move(path), std::move(data), std::move(P))
|
||||
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/", std::move(path), std::move(data),
|
||||
std::move(P))
|
||||
.release();
|
||||
};
|
||||
add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer));
|
||||
}
|
||||
|
||||
void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
|
||||
std::function<td::Status(td::FileFd&)> write_state,
|
||||
std::function<td::Status(td::FileFd &)> write_state,
|
||||
td::Promise<td::Unit> promise) {
|
||||
auto create_writer = [&](std::string path, td::Promise<std::string> P) {
|
||||
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/",
|
||||
std::move(path), std::move(write_state), std::move(P))
|
||||
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/", std::move(path),
|
||||
std::move(write_state), std::move(P))
|
||||
.release();
|
||||
};
|
||||
add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer));
|
||||
|
|
@ -941,7 +941,8 @@ void ArchiveManager::start_up() {
|
|||
void ArchiveManager::alarm() {
|
||||
alarm_timestamp() = td::Timestamp::in(60.0);
|
||||
auto stats = statistics_.to_string_and_reset();
|
||||
auto to_file_r = td::FileFd::open(db_root_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644);
|
||||
auto to_file_r =
|
||||
td::FileFd::open(db_root_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644);
|
||||
if (to_file_r.is_error()) {
|
||||
LOG(ERROR) << "Failed to open db_stats.txt: " << to_file_r.move_as_error();
|
||||
return;
|
||||
|
|
@ -1035,7 +1036,7 @@ void ArchiveManager::persistent_state_gc(std::pair<BlockSeqno, FileHash> last) {
|
|||
}
|
||||
if (res != 0) {
|
||||
delay_action([key, SelfId = actor_id(
|
||||
this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
td::Timestamp::in(1.0));
|
||||
return;
|
||||
}
|
||||
|
|
@ -1052,7 +1053,7 @@ void ArchiveManager::persistent_state_gc(std::pair<BlockSeqno, FileHash> last) {
|
|||
}
|
||||
if (!allow_delete) {
|
||||
delay_action([key, SelfId = actor_id(
|
||||
this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
td::Timestamp::in(1.0));
|
||||
return;
|
||||
}
|
||||
|
|
@ -1083,9 +1084,9 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pai
|
|||
td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore();
|
||||
perm_states_.erase(it);
|
||||
}
|
||||
delay_action([key, SelfId = actor_id(
|
||||
this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
td::Timestamp::in(1.0));
|
||||
delay_action(
|
||||
[key, SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); },
|
||||
td::Timestamp::in(1.0));
|
||||
}
|
||||
|
||||
PackageId ArchiveManager::get_temp_package_id() const {
|
||||
|
|
|
|||
|
|
@ -17,10 +17,12 @@
|
|||
Copyright 2017-2020 Telegram Systems LLP
|
||||
*/
|
||||
#include "celldb.hpp"
|
||||
|
||||
#include "files-async.hpp"
|
||||
#include "rootdb.hpp"
|
||||
|
||||
#include "td/db/RocksDb.h"
|
||||
#include "td/utils/filesystem.h"
|
||||
#include "rocksdb/utilities/optimistic_transaction_db.h"
|
||||
|
||||
#include "ton/ton-tl.hpp"
|
||||
#include "ton/ton-io.hpp"
|
||||
|
|
@ -29,7 +31,6 @@
|
|||
namespace ton {
|
||||
|
||||
namespace validator {
|
||||
|
||||
class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor {
|
||||
public:
|
||||
explicit CellDbAsyncExecutor(td::actor::ActorId<CellDbBase> cell_db) : cell_db_(std::move(cell_db)) {
|
||||
|
|
@ -38,11 +39,13 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor {
|
|||
void execute_async(std::function<void()> f) override {
|
||||
class Runner : public td::actor::Actor {
|
||||
public:
|
||||
explicit Runner(std::function<void()> f) : f_(std::move(f)) {}
|
||||
explicit Runner(std::function<void()> f) : f_(std::move(f)) {
|
||||
}
|
||||
void start_up() override {
|
||||
f_();
|
||||
stop();
|
||||
}
|
||||
|
||||
private:
|
||||
std::function<void()> f_;
|
||||
};
|
||||
|
|
@ -52,6 +55,7 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor {
|
|||
void execute_sync(std::function<void()> f) override {
|
||||
td::actor::send_closure(cell_db_, &CellDbBase::execute_sync, std::move(f));
|
||||
}
|
||||
|
||||
private:
|
||||
td::actor::ActorId<CellDbBase> cell_db_;
|
||||
};
|
||||
|
|
@ -97,13 +101,30 @@ void CellDbIn::start_up() {
|
|||
LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value());
|
||||
}
|
||||
db_options.use_direct_reads = opts_->get_celldb_direct_io();
|
||||
cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_, std::move(db_options)).move_as_ok());
|
||||
|
||||
if (opts_->get_celldb_in_memory()) {
|
||||
td::RocksDbOptions read_db_options;
|
||||
read_db_options.use_direct_reads = true;
|
||||
read_db_options.no_block_cache = true;
|
||||
read_db_options.block_cache = {};
|
||||
LOG(WARNING) << "Loading all cells in memory (because of --celldb-in-memory)";
|
||||
td::Timer timer;
|
||||
auto read_cell_db =
|
||||
std::make_shared<td::RocksDb>(td::RocksDb::open(path_, std::move(read_db_options)).move_as_ok());
|
||||
boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), {});
|
||||
in_memory_load_time_ = timer.elapsed();
|
||||
td::actor::send_closure(parent_, &CellDb::set_in_memory_boc, boc_);
|
||||
}
|
||||
|
||||
boc_ = vm::DynamicBagOfCellsDb::create();
|
||||
boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth());
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
auto rocks_db = std::make_shared<td::RocksDb>(td::RocksDb::open(path_, std::move(db_options)).move_as_ok());
|
||||
rocks_db_ = rocks_db->raw_db();
|
||||
cell_db_ = std::move(rocks_db);
|
||||
if (!opts_->get_celldb_in_memory()) {
|
||||
boc_ = vm::DynamicBagOfCellsDb::create();
|
||||
boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth());
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
}
|
||||
|
||||
alarm_timestamp() = td::Timestamp::in(10.0);
|
||||
|
||||
|
|
@ -117,31 +138,39 @@ void CellDbIn::start_up() {
|
|||
|
||||
if (opts_->get_celldb_preload_all()) {
|
||||
// Iterate whole DB in a separate thread
|
||||
delay_action([snapshot = cell_db_->snapshot()]() {
|
||||
LOG(WARNING) << "CellDb: pre-loading all keys";
|
||||
td::uint64 total = 0;
|
||||
td::Timer timer;
|
||||
auto S = snapshot->for_each([&](td::Slice, td::Slice) {
|
||||
++total;
|
||||
if (total % 1000000 == 0) {
|
||||
LOG(INFO) << "CellDb: iterated " << total << " keys";
|
||||
}
|
||||
return td::Status::OK();
|
||||
});
|
||||
if (S.is_error()) {
|
||||
LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error();
|
||||
} else {
|
||||
LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s";
|
||||
}
|
||||
}, td::Timestamp::now());
|
||||
delay_action(
|
||||
[snapshot = cell_db_->snapshot()]() {
|
||||
LOG(WARNING) << "CellDb: pre-loading all keys";
|
||||
td::uint64 total = 0;
|
||||
td::Timer timer;
|
||||
auto S = snapshot->for_each([&](td::Slice, td::Slice) {
|
||||
++total;
|
||||
if (total % 1000000 == 0) {
|
||||
LOG(INFO) << "CellDb: iterated " << total << " keys";
|
||||
}
|
||||
return td::Status::OK();
|
||||
});
|
||||
if (S.is_error()) {
|
||||
LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error();
|
||||
} else {
|
||||
LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s";
|
||||
}
|
||||
},
|
||||
td::Timestamp::now());
|
||||
}
|
||||
}
|
||||
|
||||
void CellDbIn::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
|
||||
if (opts_->get_celldb_in_memory()) {
|
||||
auto result = boc_->load_root(hash.as_slice());
|
||||
async_apply("load_cell_result", std::move(promise), std::move(result));
|
||||
return;
|
||||
}
|
||||
boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise));
|
||||
}
|
||||
|
||||
void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise) {
|
||||
TD_PERF_COUNTER(celldb_store_cell);
|
||||
td::PerfWarningTimer timer{"storecell", 0.1};
|
||||
auto key_hash = get_key_hash(block_id);
|
||||
auto R = get_block(key_hash);
|
||||
|
|
@ -181,8 +210,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promi
|
|||
set_block(key_hash, std::move(D));
|
||||
cell_db_->commit_write_batch().ensure();
|
||||
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
if (!opts_->get_celldb_in_memory()) {
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
}
|
||||
|
||||
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
|
||||
if (!opts_->get_disable_rocksdb_stats()) {
|
||||
|
|
@ -199,12 +230,50 @@ void CellDbIn::get_last_deleted_mc_state(td::Promise<BlockSeqno> promise) {
|
|||
promise.set_result(last_deleted_mc_state_);
|
||||
}
|
||||
|
||||
std::vector<std::pair<std::string, std::string>> CellDbIn::prepare_stats() {
|
||||
TD_PERF_COUNTER(celldb_prepare_stats);
|
||||
auto r_boc_stats = boc_->get_stats();
|
||||
if (r_boc_stats.is_ok()) {
|
||||
cell_db_statistics_.boc_stats_ = r_boc_stats.move_as_ok();
|
||||
}
|
||||
cell_db_statistics_.in_memory_load_time_ = in_memory_load_time_;
|
||||
auto stats = cell_db_statistics_.prepare_stats();
|
||||
auto add_stat = [&](const auto& key, const auto& value) { stats.emplace_back(key, PSTRING() << value); };
|
||||
|
||||
add_stat("started", "true");
|
||||
auto r_mem_stat = td::mem_stat();
|
||||
auto r_total_mem_stat = td::get_total_mem_stat();
|
||||
td::uint64 celldb_size = 0;
|
||||
bool ok_celldb_size = rocks_db_->GetIntProperty("rocksdb.total-sst-files-size", &celldb_size);
|
||||
if (celldb_size > 0 && r_mem_stat.is_ok() && r_total_mem_stat.is_ok() && ok_celldb_size) {
|
||||
auto mem_stat = r_mem_stat.move_as_ok();
|
||||
auto total_mem_stat = r_total_mem_stat.move_as_ok();
|
||||
add_stat("rss", td::format::as_size(mem_stat.resident_size_));
|
||||
add_stat("available_ram", td::format::as_size(total_mem_stat.available_ram));
|
||||
add_stat("total_ram", td::format::as_size(total_mem_stat.total_ram));
|
||||
add_stat("actual_ram_to_celldb_ratio", double(total_mem_stat.available_ram) / double(celldb_size));
|
||||
add_stat("if_restarted_ram_to_celldb_ratio",
|
||||
double(total_mem_stat.available_ram + mem_stat.resident_size_ - 10 * (td::uint64(1) << 30)) /
|
||||
double(celldb_size));
|
||||
add_stat("max_possible_ram_to_celldb_ratio", double(total_mem_stat.total_ram) / double(celldb_size));
|
||||
}
|
||||
|
||||
return stats;
|
||||
// do not clear statistics, it is needed for flush_db_stats
|
||||
}
|
||||
void CellDbIn::flush_db_stats() {
|
||||
if (opts_->get_disable_rocksdb_stats()) {
|
||||
return;
|
||||
}
|
||||
auto stats = td::RocksDb::statistics_to_string(statistics_) + snapshot_statistics_->to_string() +
|
||||
cell_db_statistics_.to_string();
|
||||
|
||||
auto celldb_stats = prepare_stats();
|
||||
td::StringBuilder ss;
|
||||
for (auto& [key, value] : celldb_stats) {
|
||||
ss << "ton.celldb." << key << " " << value << "\n";
|
||||
}
|
||||
|
||||
auto stats =
|
||||
td::RocksDb::statistics_to_string(statistics_) + snapshot_statistics_->to_string() + ss.as_cslice().str();
|
||||
auto to_file_r =
|
||||
td::FileFd::open(path_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644);
|
||||
if (to_file_r.is_error()) {
|
||||
|
|
@ -284,8 +353,11 @@ void CellDbIn::gc_cont(BlockHandle handle) {
|
|||
}
|
||||
|
||||
void CellDbIn::gc_cont2(BlockHandle handle) {
|
||||
TD_PERF_COUNTER(celldb_gc_cell);
|
||||
td::PerfWarningTimer timer{"gccell", 0.1};
|
||||
td::PerfWarningTimer timer_all{"gccell_all", 0.05};
|
||||
|
||||
td::PerfWarningTimer timer_get_keys{"gccell_get_keys", 0.05};
|
||||
auto key_hash = get_key_hash(handle->id());
|
||||
auto FR = get_block(key_hash);
|
||||
FR.ensure();
|
||||
|
|
@ -304,22 +376,41 @@ void CellDbIn::gc_cont2(BlockHandle handle) {
|
|||
P.prev = P.next;
|
||||
N.next = N.prev;
|
||||
}
|
||||
timer_get_keys.reset();
|
||||
|
||||
td::PerfWarningTimer timer_boc{"gccell_boc", 0.05};
|
||||
auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok();
|
||||
|
||||
boc_->dec(cell);
|
||||
boc_->prepare_commit().ensure();
|
||||
vm::CellStorer stor{*cell_db_};
|
||||
timer_boc.reset();
|
||||
|
||||
td::PerfWarningTimer timer_write_batch{"gccell_write_batch", 0.05};
|
||||
cell_db_->begin_write_batch().ensure();
|
||||
boc_->commit(stor).ensure();
|
||||
|
||||
cell_db_->erase(get_key(key_hash)).ensure();
|
||||
set_block(F.prev, std::move(P));
|
||||
set_block(F.next, std::move(N));
|
||||
cell_db_->commit_write_batch().ensure();
|
||||
alarm_timestamp() = td::Timestamp::now();
|
||||
timer_write_batch.reset();
|
||||
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
td::PerfWarningTimer timer_free_cells{"gccell_free_cells", 0.05};
|
||||
auto before = td::ref_get_delete_count();
|
||||
cell = {};
|
||||
auto after = td::ref_get_delete_count();
|
||||
if (timer_free_cells.elapsed() > 0.04) {
|
||||
LOG(ERROR) << "deleted " << after - before << " cells";
|
||||
}
|
||||
timer_free_cells.reset();
|
||||
|
||||
td::PerfWarningTimer timer_finish{"gccell_finish", 0.05};
|
||||
if (!opts_->get_celldb_in_memory()) {
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
}
|
||||
|
||||
DCHECK(get_block(key_hash).is_error());
|
||||
if (!opts_->get_disable_rocksdb_stats()) {
|
||||
|
|
@ -329,6 +420,8 @@ void CellDbIn::gc_cont2(BlockHandle handle) {
|
|||
last_deleted_mc_state_ = handle->id().seqno();
|
||||
}
|
||||
LOG(DEBUG) << "Deleted state " << handle->id().to_str();
|
||||
timer_finish.reset();
|
||||
timer_all.reset();
|
||||
}
|
||||
|
||||
void CellDbIn::skip_gc() {
|
||||
|
|
@ -401,7 +494,7 @@ void CellDbIn::migrate_cells() {
|
|||
boc_->set_loader(std::make_unique<vm::CellLoader>(*loader)).ensure();
|
||||
cell_db_->begin_write_batch().ensure();
|
||||
td::uint32 checked = 0, migrated = 0;
|
||||
for (auto it = cells_to_migrate_.begin(); it != cells_to_migrate_.end() && checked < 128; ) {
|
||||
for (auto it = cells_to_migrate_.begin(); it != cells_to_migrate_.end() && checked < 128;) {
|
||||
++checked;
|
||||
td::Bits256 hash = *it;
|
||||
it = cells_to_migrate_.erase(it);
|
||||
|
|
@ -438,7 +531,32 @@ void CellDbIn::migrate_cells() {
|
|||
}
|
||||
}
|
||||
|
||||
void CellDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
|
||||
promise.set_value(decltype(prepared_stats_)(prepared_stats_));
|
||||
}
|
||||
|
||||
void CellDb::update_stats(td::Result<std::vector<std::pair<std::string, std::string>>> r_stats) {
|
||||
if (r_stats.is_error()) {
|
||||
LOG(ERROR) << "error updating stats: " << r_stats.error();
|
||||
} else {
|
||||
prepared_stats_ = r_stats.move_as_ok();
|
||||
}
|
||||
alarm_timestamp() = td::Timestamp::in(2.0);
|
||||
}
|
||||
|
||||
void CellDb::alarm() {
|
||||
send_closure(cell_db_, &CellDbIn::prepare_stats, td::promise_send_closure(actor_id(this), &CellDb::update_stats));
|
||||
}
|
||||
|
||||
void CellDb::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
|
||||
if (in_memory_boc_) {
|
||||
auto result = in_memory_boc_->load_root_thread_safe(hash.as_slice());
|
||||
if (result.is_ok()) {
|
||||
return async_apply("load_cell_result", std::move(promise), std::move(result));
|
||||
} else {
|
||||
LOG(ERROR) << "load_root_thread_safe failed - this is suspicious";
|
||||
}
|
||||
}
|
||||
if (!started_) {
|
||||
td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise));
|
||||
} else {
|
||||
|
|
@ -496,12 +614,24 @@ td::BufferSlice CellDbIn::DbEntry::release() {
|
|||
return create_serialize_tl_object<ton_api::db_celldb_value>(create_tl_block_id(block_id), prev, next, root_hash);
|
||||
}
|
||||
|
||||
std::string CellDbIn::CellDbStatistics::to_string() {
|
||||
td::StringBuilder ss;
|
||||
ss << "ton.celldb.store_cell.micros " << store_cell_time_.to_string() << "\n";
|
||||
ss << "ton.celldb.gc_cell.micros " << gc_cell_time_.to_string() << "\n";
|
||||
ss << "ton.celldb.total_time.micros : " << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6 << "\n";
|
||||
return ss.as_cslice().str();
|
||||
std::vector<std::pair<std::string, std::string>> CellDbIn::CellDbStatistics::prepare_stats() {
|
||||
std::vector<std::pair<std::string, std::string>> stats;
|
||||
stats.emplace_back("store_cell.micros", PSTRING() << store_cell_time_.to_string());
|
||||
stats.emplace_back("gc_cell.micros", PSTRING() << gc_cell_time_.to_string());
|
||||
stats.emplace_back("total_time.micros", PSTRING() << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6);
|
||||
stats.emplace_back("in_memory", PSTRING() << bool(in_memory_load_time_));
|
||||
if (in_memory_load_time_) {
|
||||
stats.emplace_back("in_memory_load_time", PSTRING() << in_memory_load_time_.value());
|
||||
}
|
||||
if (boc_stats_) {
|
||||
stats.emplace_back("cells_count", PSTRING() << boc_stats_->cells_total_count);
|
||||
stats.emplace_back("cells_size", PSTRING() << boc_stats_->cells_total_size);
|
||||
stats.emplace_back("roots_count", PSTRING() << boc_stats_->roots_total_count);
|
||||
for (auto& [key, value] : boc_stats_->custom_stats) {
|
||||
stats.emplace_back(key, value);
|
||||
}
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
} // namespace validator
|
||||
|
|
|
|||
|
|
@ -29,9 +29,12 @@
|
|||
#include "db-utils.h"
|
||||
#include "td/db/RocksDb.h"
|
||||
|
||||
#include <optional>
|
||||
|
||||
namespace rocksdb {
|
||||
class Statistics;
|
||||
}
|
||||
class DB;
|
||||
} // namespace rocksdb
|
||||
|
||||
namespace ton {
|
||||
|
||||
|
|
@ -58,6 +61,7 @@ class CellDbIn : public CellDbBase {
|
|||
public:
|
||||
using KeyHash = td::Bits256;
|
||||
|
||||
std::vector<std::pair<std::string, std::string>> prepare_stats();
|
||||
void load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise);
|
||||
void store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise);
|
||||
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise);
|
||||
|
|
@ -111,13 +115,15 @@ class CellDbIn : public CellDbBase {
|
|||
std::string path_;
|
||||
td::Ref<ValidatorManagerOptions> opts_;
|
||||
|
||||
std::unique_ptr<vm::DynamicBagOfCellsDb> boc_;
|
||||
std::shared_ptr<vm::DynamicBagOfCellsDb> boc_;
|
||||
std::shared_ptr<vm::KeyValue> cell_db_;
|
||||
std::shared_ptr<rocksdb::DB> rocks_db_;
|
||||
|
||||
std::function<void(const vm::CellLoader::LoadResult&)> on_load_callback_;
|
||||
std::set<td::Bits256> cells_to_migrate_;
|
||||
td::Timestamp migrate_after_ = td::Timestamp::never();
|
||||
bool migration_active_ = false;
|
||||
std::optional<double> in_memory_load_time_;
|
||||
|
||||
struct MigrationStats {
|
||||
td::Timer start_;
|
||||
|
|
@ -133,8 +139,10 @@ class CellDbIn : public CellDbBase {
|
|||
PercentileStats store_cell_time_;
|
||||
PercentileStats gc_cell_time_;
|
||||
td::Timestamp stats_start_time_ = td::Timestamp::now();
|
||||
std::optional<double> in_memory_load_time_;
|
||||
std::optional<vm::DynamicBagOfCellsDb::Stats> boc_stats_;
|
||||
|
||||
std::string to_string();
|
||||
std::vector<std::pair<std::string, std::string>> prepare_stats();
|
||||
void clear() {
|
||||
*this = CellDbStatistics{};
|
||||
}
|
||||
|
|
@ -162,12 +170,25 @@ class CellDbIn : public CellDbBase {
|
|||
|
||||
class CellDb : public CellDbBase {
|
||||
public:
|
||||
void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise);
|
||||
void load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise);
|
||||
void store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise);
|
||||
void update_snapshot(std::unique_ptr<td::KeyValueReader> snapshot) {
|
||||
CHECK(!opts_->get_celldb_in_memory());
|
||||
if (!started_) {
|
||||
alarm();
|
||||
}
|
||||
started_ = true;
|
||||
boc_->set_loader(std::make_unique<vm::CellLoader>(std::move(snapshot), on_load_callback_)).ensure();
|
||||
}
|
||||
void set_in_memory_boc(std::shared_ptr<const vm::DynamicBagOfCellsDb> in_memory_boc) {
|
||||
CHECK(opts_->get_celldb_in_memory());
|
||||
if (!started_) {
|
||||
alarm();
|
||||
}
|
||||
started_ = true;
|
||||
in_memory_boc_ = std::move(in_memory_boc);
|
||||
}
|
||||
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise);
|
||||
void get_last_deleted_mc_state(td::Promise<BlockSeqno> promise);
|
||||
|
||||
|
|
@ -185,9 +206,14 @@ class CellDb : public CellDbBase {
|
|||
td::actor::ActorOwn<CellDbIn> cell_db_;
|
||||
|
||||
std::unique_ptr<vm::DynamicBagOfCellsDb> boc_;
|
||||
std::shared_ptr<const vm::DynamicBagOfCellsDb> in_memory_boc_;
|
||||
bool started_ = false;
|
||||
std::vector<std::pair<std::string, std::string>> prepared_stats_{{"started", "false"}};
|
||||
|
||||
std::function<void(const vm::CellLoader::LoadResult&)> on_load_callback_;
|
||||
|
||||
void update_stats(td::Result<std::vector<std::pair<std::string, std::string>>> stats);
|
||||
void alarm() override;
|
||||
};
|
||||
|
||||
} // namespace validator
|
||||
|
|
|
|||
|
|
@ -79,14 +79,14 @@ std::string Block::filename() const {
|
|||
|
||||
std::string Block::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "block_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string BlockShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "block_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
@ -116,14 +116,14 @@ std::string PersistentState::filename() const {
|
|||
|
||||
std::string PersistentState::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "state_" << masterchain_block_id.seqno() << "_" << block_id.id.workchain << "_" << s << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string PersistentStateShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(shard_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(shard_id.shard));
|
||||
return PSTRING() << "state_" << masterchain_seqno << "_" << shard_id.workchain << "_" << s << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
@ -137,14 +137,14 @@ std::string Proof::filename() const {
|
|||
|
||||
std::string Proof::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "proof_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string ProofShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "proof_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
@ -158,14 +158,14 @@ std::string ProofLink::filename() const {
|
|||
|
||||
std::string ProofLink::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "prooflink_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string ProofLinkShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "prooflink_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
@ -179,14 +179,14 @@ std::string Signatures::filename() const {
|
|||
|
||||
std::string Signatures::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "signatures_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string SignaturesShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "signatures_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
|
@ -202,14 +202,14 @@ std::string Candidate::filename() const {
|
|||
|
||||
std::string Candidate::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "candidate_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string CandidateShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "candidate_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
@ -223,14 +223,14 @@ std::string CandidateRef::filename() const {
|
|||
|
||||
std::string CandidateRef::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "candidateref_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string CandidateRefShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "candidateref_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
|
@ -245,14 +245,14 @@ std::string BlockInfo::filename() const {
|
|||
|
||||
std::string BlockInfo::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.id.shard));
|
||||
return PSTRING() << "info_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
|
||||
<< hash().to_hex();
|
||||
}
|
||||
|
||||
std::string BlockInfoShort::filename_short() const {
|
||||
char s[33];
|
||||
sprintf(s, "%llx", static_cast<long long>(block_id.shard));
|
||||
snprintf(s, sizeof(s), "%llx", static_cast<long long>(block_id.shard));
|
||||
return PSTRING() << "info_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -441,6 +441,7 @@ void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise<bool> promise) {
|
|||
|
||||
void RootDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
|
||||
auto merger = StatsMerger::create(std::move(promise));
|
||||
td::actor::send_closure(cell_db_, &CellDb::prepare_stats, merger.make_promise("celldb."));
|
||||
}
|
||||
|
||||
void RootDb::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise) {
|
||||
|
|
|
|||
|
|
@ -244,6 +244,7 @@ void WaitBlockState::got_block_data(td::Ref<BlockData> data) {
|
|||
}
|
||||
|
||||
void WaitBlockState::apply() {
|
||||
TD_PERF_COUNTER(apply_block_to_state);
|
||||
td::PerfWarningTimer t{"applyblocktostate", 0.1};
|
||||
auto S = prev_state_.write().apply_block(handle_->id(), block_);
|
||||
if (S.is_error()) {
|
||||
|
|
|
|||
|
|
@ -4932,11 +4932,11 @@ bool Collator::check_block_overload() {
|
|||
}
|
||||
char buffer[17];
|
||||
if (history_weight(overload_history_) >= 0) {
|
||||
sprintf(buffer, "%016llx", (unsigned long long)overload_history_);
|
||||
snprintf(buffer, sizeof(buffer), "%016llx", (unsigned long long)overload_history_);
|
||||
LOG(INFO) << "want_split set because of overload history " << buffer;
|
||||
want_split_ = true;
|
||||
} else if (history_weight(underload_history_) >= 0) {
|
||||
sprintf(buffer, "%016llx", (unsigned long long)underload_history_);
|
||||
snprintf(buffer, sizeof(buffer), "%016llx", (unsigned long long)underload_history_);
|
||||
LOG(INFO) << "want_merge set because of underload history " << buffer;
|
||||
want_merge_ = true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1860,7 +1860,7 @@ void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector<
|
|||
request_mc_block_data_state(blkid);
|
||||
} else {
|
||||
// get configuration from previous key block
|
||||
load_prevKeyBlock(blkid, [this, blkid, mode, param_list = std::move(param_list)](
|
||||
load_prevKeyBlock(blkid, [this, mode, param_list = std::move(param_list)](
|
||||
td::Result<std::pair<BlockIdExt, Ref<BlockQ>>> res) mutable {
|
||||
if (res.is_error()) {
|
||||
this->abort_query(res.move_as_error());
|
||||
|
|
@ -2057,7 +2057,7 @@ void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid,
|
|||
|
||||
ton::AccountIdPrefixFull pfx{blkid.workchain, blkid.shard};
|
||||
auto P = td::PromiseCreator::lambda(
|
||||
[Self = actor_id(this), mc_blkid, manager = manager_, mode, pfx](td::Result<ConstBlockHandle> res) {
|
||||
[Self = actor_id(this), mc_blkid, manager = manager_, pfx](td::Result<ConstBlockHandle> res) {
|
||||
if (res.is_error()) {
|
||||
td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
|
||||
return;
|
||||
|
|
@ -2073,7 +2073,7 @@ void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid,
|
|||
}
|
||||
LOG(DEBUG) << "requesting data for block " << handle->id().to_str();
|
||||
td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle,
|
||||
[Self, mc_ref_blkid = handle->masterchain_ref_block(), mc_blkid, pfx, mode](td::Result<Ref<BlockData>> res) {
|
||||
[Self, mc_ref_blkid = handle->masterchain_ref_block(), pfx](td::Result<Ref<BlockData>> res) {
|
||||
if (res.is_error()) {
|
||||
td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -290,6 +290,7 @@ td::Result<std::pair<td::Ref<ShardState>, td::Ref<ShardState>>> ShardStateQ::spl
|
|||
}
|
||||
|
||||
td::Result<td::BufferSlice> ShardStateQ::serialize() const {
|
||||
TD_PERF_COUNTER(serialize_state);
|
||||
td::PerfWarningTimer perf_timer_{"serializestate", 0.1};
|
||||
if (!data.is_null()) {
|
||||
return data.clone();
|
||||
|
|
@ -314,6 +315,7 @@ td::Result<td::BufferSlice> ShardStateQ::serialize() const {
|
|||
}
|
||||
|
||||
td::Status ShardStateQ::serialize_to_file(td::FileFd& fd) const {
|
||||
TD_PERF_COUNTER(serialize_state_to_file);
|
||||
td::PerfWarningTimer perf_timer_{"serializestate", 0.1};
|
||||
if (!data.is_null()) {
|
||||
auto cur_data = data.clone();
|
||||
|
|
|
|||
|
|
@ -382,6 +382,10 @@ class ValidatorManagerImpl : public ValidatorManager {
|
|||
UNREACHABLE();
|
||||
}
|
||||
|
||||
void prepare_actor_stats(td::Promise<std::string> promise) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
|
||||
void prepare_perf_timer_stats(td::Promise<std::vector<PerfTimerStats>> promise) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -445,6 +445,10 @@ class ValidatorManagerImpl : public ValidatorManager {
|
|||
UNREACHABLE();
|
||||
}
|
||||
|
||||
void prepare_actor_stats(td::Promise<std::string> promise) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
|
||||
void prepare_perf_timer_stats(td::Promise<std::vector<PerfTimerStats>> promise) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1780,6 +1780,7 @@ void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, Sh
|
|||
|
||||
void ValidatorManagerImpl::start_up() {
|
||||
db_ = create_db_actor(actor_id(this), db_root_, opts_);
|
||||
actor_stats_ = td::actor::create_actor<td::actor::ActorStats>("actor_stats");
|
||||
lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_);
|
||||
token_manager_ = td::actor::create_actor<TokenManager>("tokenmanager");
|
||||
td::mkdir(db_root_ + "/tmp/").ensure();
|
||||
|
|
@ -2332,8 +2333,8 @@ void ValidatorManagerImpl::update_shards() {
|
|||
new_next_validator_groups_.emplace(val_group_id, std::move(it->second));
|
||||
} else {
|
||||
new_next_validator_groups_.emplace(
|
||||
val_group_id,
|
||||
ValidatorGroupEntry{create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard});
|
||||
val_group_id, ValidatorGroupEntry{
|
||||
create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2989,6 +2990,10 @@ void ValidatorManagerImpl::send_peek_key_block_request() {
|
|||
send_get_next_key_blocks_request(last_known_key_block_handle_->id(), 1, std::move(P));
|
||||
}
|
||||
|
||||
void ValidatorManagerImpl::prepare_actor_stats(td::Promise<std::string> promise) {
|
||||
send_closure(actor_stats_, &td::actor::ActorStats::prepare_stats, std::move(promise));
|
||||
}
|
||||
|
||||
void ValidatorManagerImpl::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) {
|
||||
auto merger = StatsMerger::create(std::move(promise));
|
||||
|
||||
|
|
@ -3021,6 +3026,9 @@ void ValidatorManagerImpl::prepare_stats(td::Promise<std::vector<std::pair<std::
|
|||
promise.set_value(std::move(vec));
|
||||
});
|
||||
}
|
||||
td::NamedThreadSafeCounter::get_default().for_each([&](auto key, auto value) {
|
||||
vec.emplace_back("counter." + key, PSTRING() << value);
|
||||
});
|
||||
|
||||
if (!shard_client_.empty()) {
|
||||
auto P = td::PromiseCreator::lambda([promise = merger.make_promise("")](td::Result<BlockSeqno> R) mutable {
|
||||
|
|
@ -3244,18 +3252,18 @@ void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id,
|
|||
promise.set_result(R.move_as_ok());
|
||||
return;
|
||||
}
|
||||
td::actor::send_closure(manager, &ValidatorManagerImpl::get_block_handle_for_litequery,
|
||||
block_id, [manager, promise = std::move(promise)](td::Result<ConstBlockHandle> R) mutable {
|
||||
TRY_RESULT_PROMISE(promise, handle, std::move(R));
|
||||
td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, std::move(handle),
|
||||
std::move(promise));
|
||||
});
|
||||
td::actor::send_closure(manager, &ValidatorManagerImpl::get_block_handle_for_litequery, block_id,
|
||||
[manager, promise = std::move(promise)](td::Result<ConstBlockHandle> R) mutable {
|
||||
TRY_RESULT_PROMISE(promise, handle, std::move(R));
|
||||
td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db,
|
||||
std::move(handle), std::move(promise));
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void ValidatorManagerImpl::get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt,
|
||||
td::Promise<ConstBlockHandle> promise) {
|
||||
td::Promise<ConstBlockHandle> promise) {
|
||||
get_block_by_lt_from_db(
|
||||
account, lt, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result<ConstBlockHandle> R) mutable {
|
||||
if (R.is_ok() && R.ok()->is_applied()) {
|
||||
|
|
@ -3268,7 +3276,7 @@ void ValidatorManagerImpl::get_block_by_lt_for_litequery(AccountIdPrefixFull acc
|
|||
}
|
||||
|
||||
void ValidatorManagerImpl::get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts,
|
||||
td::Promise<ConstBlockHandle> promise) {
|
||||
td::Promise<ConstBlockHandle> promise) {
|
||||
get_block_by_unix_time_from_db(
|
||||
account, ts, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result<ConstBlockHandle> R) mutable {
|
||||
if (R.is_ok() && R.ok()->is_applied()) {
|
||||
|
|
@ -3281,7 +3289,7 @@ void ValidatorManagerImpl::get_block_by_unix_time_for_litequery(AccountIdPrefixF
|
|||
}
|
||||
|
||||
void ValidatorManagerImpl::get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno,
|
||||
td::Promise<ConstBlockHandle> promise) {
|
||||
td::Promise<ConstBlockHandle> promise) {
|
||||
get_block_by_seqno_from_db(
|
||||
account, seqno,
|
||||
[=, SelfId = actor_id(this), promise = std::move(promise)](td::Result<ConstBlockHandle> R) mutable {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@
|
|||
#include "common/refcnt.hpp"
|
||||
#include "interfaces/validator-manager.h"
|
||||
#include "interfaces/db.h"
|
||||
#include "td/actor/ActorStats.h"
|
||||
#include "td/actor/PromiseFuture.h"
|
||||
#include "td/utils/SharedSlice.h"
|
||||
#include "td/utils/buffer.h"
|
||||
|
|
@ -611,6 +612,8 @@ class ValidatorManagerImpl : public ValidatorManager {
|
|||
|
||||
void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) override;
|
||||
|
||||
void prepare_actor_stats(td::Promise<std::string> promise) override;
|
||||
|
||||
void prepare_perf_timer_stats(td::Promise<std::vector<PerfTimerStats>> promise) override;
|
||||
void add_perf_timer_stat(std::string name, double duration) override;
|
||||
|
||||
|
|
@ -716,6 +719,7 @@ class ValidatorManagerImpl : public ValidatorManager {
|
|||
private:
|
||||
std::unique_ptr<Callback> callback_;
|
||||
td::actor::ActorOwn<Db> db_;
|
||||
td::actor::ActorOwn<td::actor::ActorStats> actor_stats_;
|
||||
|
||||
bool started_ = false;
|
||||
bool allow_validate_ = false;
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@
|
|||
#include "ton/ton-io.hpp"
|
||||
#include "common/delay.h"
|
||||
#include "td/utils/filesystem.h"
|
||||
#include "td/utils/HashSet.h"
|
||||
|
||||
namespace ton {
|
||||
|
||||
|
|
@ -256,17 +257,17 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) {
|
|||
class CachedCellDbReader : public vm::CellDbReader {
|
||||
public:
|
||||
CachedCellDbReader(std::shared_ptr<vm::CellDbReader> parent,
|
||||
std::shared_ptr<std::map<td::Bits256, td::Ref<vm::Cell>>> cache)
|
||||
std::shared_ptr<vm::CellHashSet> cache)
|
||||
: parent_(std::move(parent)), cache_(std::move(cache)) {
|
||||
}
|
||||
td::Result<td::Ref<vm::DataCell>> load_cell(td::Slice hash) override {
|
||||
++total_reqs_;
|
||||
DCHECK(hash.size() == 32);
|
||||
if (cache_) {
|
||||
auto it = cache_->find(td::Bits256{(const unsigned char*)hash.data()});
|
||||
auto it = cache_->find(hash);
|
||||
if (it != cache_->end()) {
|
||||
++cached_reqs_;
|
||||
TRY_RESULT(loaded_cell, it->second->load_cell());
|
||||
TRY_RESULT(loaded_cell, (*it)->load_cell());
|
||||
return loaded_cell.data_cell;
|
||||
}
|
||||
}
|
||||
|
|
@ -277,7 +278,7 @@ class CachedCellDbReader : public vm::CellDbReader {
|
|||
}
|
||||
private:
|
||||
std::shared_ptr<vm::CellDbReader> parent_;
|
||||
std::shared_ptr<std::map<td::Bits256, td::Ref<vm::Cell>>> cache_;
|
||||
std::shared_ptr<vm::CellHashSet> cache_;
|
||||
|
||||
td::uint64 total_reqs_ = 0;
|
||||
td::uint64 cached_reqs_ = 0;
|
||||
|
|
@ -301,10 +302,9 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard)
|
|||
td::Timer timer;
|
||||
LOG(WARNING) << "Preloading previous persistent state for shard " << shard.to_str() << " ("
|
||||
<< cur_shards.size() << " files)";
|
||||
std::map<td::Bits256, td::Ref<vm::Cell>> cells;
|
||||
vm::CellHashSet cells;
|
||||
std::function<void(td::Ref<vm::Cell>)> dfs = [&](td::Ref<vm::Cell> cell) {
|
||||
td::Bits256 hash = cell->get_hash().bits();
|
||||
if (!cells.emplace(hash, cell).second) {
|
||||
if (!cells.insert(cell).second) {
|
||||
return;
|
||||
}
|
||||
bool is_special;
|
||||
|
|
@ -332,7 +332,7 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard)
|
|||
dfs(r_root.move_as_ok());
|
||||
}
|
||||
LOG(WARNING) << "Preloaded previous state: " << cells.size() << " cells in " << timer.elapsed() << "s";
|
||||
cache = std::make_shared<std::map<td::Bits256, td::Ref<vm::Cell>>>(std::move(cells));
|
||||
cache = std::make_shared<vm::CellHashSet>(std::move(cells));
|
||||
}
|
||||
|
||||
void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state,
|
||||
|
|
@ -353,15 +353,18 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state
|
|||
}
|
||||
}
|
||||
|
||||
auto write_data = [shard = state->get_shard(), hash = state->root_cell()->get_hash(), cell_db_reader,
|
||||
auto write_data = [shard = state->get_shard(), root = state->root_cell(), cell_db_reader,
|
||||
previous_state_cache = previous_state_cache_,
|
||||
fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(),
|
||||
cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable {
|
||||
if (!cell_db_reader) {
|
||||
return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token));
|
||||
}
|
||||
if (fast_serializer_enabled) {
|
||||
previous_state_cache->prepare_cache(shard);
|
||||
}
|
||||
auto new_cell_db_reader = std::make_shared<CachedCellDbReader>(cell_db_reader, previous_state_cache->cache);
|
||||
auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, hash, fd, 31, std::move(cancellation_token));
|
||||
auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token));
|
||||
new_cell_db_reader->print_stats();
|
||||
return res;
|
||||
};
|
||||
|
|
@ -415,15 +418,18 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardStat
|
|||
return;
|
||||
}
|
||||
LOG(ERROR) << "serializing shard state " << handle->id().id.to_str();
|
||||
auto write_data = [shard = state->get_shard(), hash = state->root_cell()->get_hash(), cell_db_reader,
|
||||
auto write_data = [shard = state->get_shard(), root = state->root_cell(), cell_db_reader,
|
||||
previous_state_cache = previous_state_cache_,
|
||||
fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(),
|
||||
cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable {
|
||||
if (!cell_db_reader) {
|
||||
return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token));
|
||||
}
|
||||
if (fast_serializer_enabled) {
|
||||
previous_state_cache->prepare_cache(shard);
|
||||
}
|
||||
auto new_cell_db_reader = std::make_shared<CachedCellDbReader>(cell_db_reader, previous_state_cache->cache);
|
||||
auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, hash, fd, 31, std::move(cancellation_token));
|
||||
auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token));
|
||||
new_cell_db_reader->print_stats();
|
||||
return res;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ class AsyncStateSerializer : public td::actor::Actor {
|
|||
std::vector<BlockIdExt> shards_;
|
||||
struct PreviousStateCache {
|
||||
std::vector<std::pair<std::string, ShardIdFull>> state_files;
|
||||
std::shared_ptr<std::map<td::Bits256, td::Ref<vm::Cell>>> cache;
|
||||
std::shared_ptr<vm::CellHashSet> cache;
|
||||
std::vector<ShardIdFull> cur_shards;
|
||||
|
||||
void prepare_cache(ShardIdFull shard);
|
||||
|
|
|
|||
|
|
@ -136,6 +136,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
|
|||
bool get_celldb_preload_all() const override {
|
||||
return celldb_preload_all_;
|
||||
}
|
||||
bool get_celldb_in_memory() const override {
|
||||
return celldb_in_memory_;
|
||||
}
|
||||
td::optional<double> get_catchain_max_block_delay() const override {
|
||||
return catchain_max_block_delay_;
|
||||
}
|
||||
|
|
@ -231,6 +234,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
|
|||
void set_celldb_preload_all(bool value) override {
|
||||
celldb_preload_all_ = value;
|
||||
}
|
||||
void set_celldb_in_memory(bool value) override {
|
||||
celldb_in_memory_ = value;
|
||||
}
|
||||
void set_catchain_max_block_delay(double value) override {
|
||||
catchain_max_block_delay_ = value;
|
||||
}
|
||||
|
|
@ -297,6 +303,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
|
|||
td::optional<td::uint64> celldb_cache_size_;
|
||||
bool celldb_direct_io_ = false;
|
||||
bool celldb_preload_all_ = false;
|
||||
bool celldb_in_memory_ = false;
|
||||
td::optional<double> catchain_max_block_delay_, catchain_max_block_delay_slow_;
|
||||
bool state_serializer_enabled_ = true;
|
||||
td::Ref<CollatorOptions> collator_options_{true};
|
||||
|
|
|
|||
|
|
@ -116,6 +116,7 @@ struct ValidatorManagerOptions : public td::CntObject {
|
|||
virtual BlockSeqno sync_upto() const = 0;
|
||||
virtual std::string get_session_logs_file() const = 0;
|
||||
virtual td::uint32 get_celldb_compress_depth() const = 0;
|
||||
virtual bool get_celldb_in_memory() const = 0;
|
||||
virtual size_t get_max_open_archive_files() const = 0;
|
||||
virtual double get_archive_preload_period() const = 0;
|
||||
virtual bool get_disable_rocksdb_stats() const = 0;
|
||||
|
|
@ -155,6 +156,7 @@ struct ValidatorManagerOptions : public td::CntObject {
|
|||
virtual void set_celldb_cache_size(td::uint64 value) = 0;
|
||||
virtual void set_celldb_direct_io(bool value) = 0;
|
||||
virtual void set_celldb_preload_all(bool value) = 0;
|
||||
virtual void set_celldb_in_memory(bool value) = 0;
|
||||
virtual void set_catchain_max_block_delay(double value) = 0;
|
||||
virtual void set_catchain_max_block_delay_slow(double value) = 0;
|
||||
virtual void set_state_serializer_enabled(bool value) = 0;
|
||||
|
|
@ -302,6 +304,7 @@ class ValidatorManagerInterface : public td::actor::Actor {
|
|||
|
||||
virtual void run_ext_query(td::BufferSlice data, td::Promise<td::BufferSlice> promise) = 0;
|
||||
virtual void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) = 0;
|
||||
virtual void prepare_actor_stats(td::Promise<std::string> promise) = 0;
|
||||
|
||||
virtual void prepare_perf_timer_stats(td::Promise<std::vector<PerfTimerStats>> promise) = 0;
|
||||
virtual void add_perf_timer_stat(std::string name, double duration) = 0;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue