mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
Improve tweaking for high throughput (#610)
* Option "--disable-ext-msg-broadcast" * "Get shard out queue size" query * Move disabling ext msg broadcasts from command-line arguments to config * Fix compilation error * Asynchronous store_cell and gc in celldb * Make GC in celldb work evenly over time * Increase timeouts for downloading blocks * Reuse blocks from previous rounds in validator session * Use Rldp2 in FullNode for downloading persistent states and archives * Improve logs in download-archive-slice and download-state * Decrease delay between serializing shards * Make CellDbIn::load_cell synchronous to avoid interfering with store_cell --------- Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK
GitHub
parent
30c742aedd
commit
47311d6e0e
33 changed files with 712 additions and 163 deletions
|
|
@ -33,11 +33,11 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor {
|
|||
explicit CellDbAsyncExecutor(td::actor::ActorId<CellDbBase> cell_db) : cell_db_(std::move(cell_db)) {
|
||||
}
|
||||
|
||||
void execute_async(std::function<void()> f) {
|
||||
void execute_async(std::function<void()> f) override {
|
||||
class Runner : public td::actor::Actor {
|
||||
public:
|
||||
explicit Runner(std::function<void()> f) : f_(std::move(f)) {}
|
||||
void start_up() {
|
||||
void start_up() override {
|
||||
f_();
|
||||
stop();
|
||||
}
|
||||
|
|
@ -47,7 +47,7 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor {
|
|||
td::actor::create_actor<Runner>("executeasync", std::move(f)).release();
|
||||
}
|
||||
|
||||
void execute_sync(std::function<void()> f) {
|
||||
void execute_sync(std::function<void()> f) override {
|
||||
td::actor::send_closure(cell_db_, &CellDbBase::execute_sync, std::move(f));
|
||||
}
|
||||
private:
|
||||
|
|
@ -83,23 +83,45 @@ void CellDbIn::start_up() {
|
|||
set_block(empty, std::move(e));
|
||||
cell_db_->commit_write_batch().ensure();
|
||||
}
|
||||
last_gc_ = empty;
|
||||
}
|
||||
|
||||
void CellDbIn::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
|
||||
boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise));
|
||||
enqueue([this, hash, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
|
||||
if (R.is_error()) {
|
||||
return;
|
||||
}
|
||||
promise.set_result(boc_->load_cell(hash.as_slice()));
|
||||
release_db();
|
||||
});
|
||||
}
|
||||
|
||||
void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise) {
|
||||
td::PerfWarningTimer timer{"storecell", 0.1};
|
||||
auto key_hash = get_key_hash(block_id);
|
||||
auto R = get_block(key_hash);
|
||||
// duplicate
|
||||
if (R.is_ok()) {
|
||||
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
|
||||
return;
|
||||
}
|
||||
enqueue([this, block_id, cell = std::move(cell), promise = std::move(promise)](td::Result<td::Unit> R0) mutable {
|
||||
if (R0.is_error()) {
|
||||
return;
|
||||
}
|
||||
promise = promise.wrap([timer = td::PerfWarningTimer{"storecell", 0.1}](td::Ref<vm::DataCell> &&r) { return r; });
|
||||
auto key_hash = get_key_hash(block_id);
|
||||
auto R = get_block(key_hash);
|
||||
// duplicate
|
||||
if (R.is_ok()) {
|
||||
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
|
||||
release_db();
|
||||
return;
|
||||
}
|
||||
|
||||
boc_->inc(cell);
|
||||
boc_->prepare_commit_async(
|
||||
async_executor, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::Unit> R) mutable {
|
||||
R.ensure();
|
||||
td::actor::send_closure(SelfId, &CellDbIn::store_cell_cont, block_id, cell, std::move(promise));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell,
|
||||
td::Promise<td::Ref<vm::DataCell>> promise) {
|
||||
auto key_hash = get_key_hash(block_id);
|
||||
auto empty = get_empty_key_hash();
|
||||
auto ER = get_block(empty);
|
||||
ER.ensure();
|
||||
|
|
@ -120,9 +142,7 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promi
|
|||
P.prev = key_hash;
|
||||
}
|
||||
|
||||
boc_->inc(cell);
|
||||
boc_->prepare_commit().ensure();
|
||||
vm::CellStorer stor{*cell_db_.get()};
|
||||
vm::CellStorer stor{*cell_db_};
|
||||
cell_db_->begin_write_batch().ensure();
|
||||
boc_->commit(stor).ensure();
|
||||
set_block(empty, std::move(E));
|
||||
|
|
@ -134,24 +154,29 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promi
|
|||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
|
||||
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
|
||||
release_db();
|
||||
}
|
||||
|
||||
void CellDbIn::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) {
|
||||
promise.set_result(boc_->get_cell_db_reader());
|
||||
enqueue([this, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
|
||||
if (R.is_error()) {
|
||||
return;
|
||||
}
|
||||
promise.set_result(boc_->get_cell_db_reader());
|
||||
release_db();
|
||||
});
|
||||
}
|
||||
|
||||
void CellDbIn::alarm() {
|
||||
auto R = get_block(last_gc_);
|
||||
R.ensure();
|
||||
|
||||
auto N = R.move_as_ok();
|
||||
auto E = get_block(get_empty_key_hash()).move_as_ok();
|
||||
auto N = get_block(E.next).move_as_ok();
|
||||
if (N.is_empty()) {
|
||||
last_gc_ = N.next;
|
||||
alarm_timestamp() = td::Timestamp::in(0.1);
|
||||
return;
|
||||
}
|
||||
|
||||
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<bool> R) {
|
||||
auto block_id = N.block_id;
|
||||
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id](td::Result<bool> R) {
|
||||
if (R.is_error()) {
|
||||
td::actor::send_closure(SelfId, &CellDbIn::skip_gc);
|
||||
} else {
|
||||
|
|
@ -159,24 +184,19 @@ void CellDbIn::alarm() {
|
|||
if (!value) {
|
||||
td::actor::send_closure(SelfId, &CellDbIn::skip_gc);
|
||||
} else {
|
||||
td::actor::send_closure(SelfId, &CellDbIn::gc);
|
||||
td::actor::send_closure(SelfId, &CellDbIn::gc, block_id);
|
||||
}
|
||||
}
|
||||
});
|
||||
td::actor::send_closure(root_db_, &RootDb::allow_state_gc, N.block_id, std::move(P));
|
||||
td::actor::send_closure(root_db_, &RootDb::allow_state_gc, block_id, std::move(P));
|
||||
}
|
||||
|
||||
void CellDbIn::gc() {
|
||||
auto R = get_block(last_gc_);
|
||||
R.ensure();
|
||||
|
||||
auto N = R.move_as_ok();
|
||||
|
||||
void CellDbIn::gc(BlockIdExt block_id) {
|
||||
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
|
||||
R.ensure();
|
||||
td::actor::send_closure(SelfId, &CellDbIn::gc_cont, R.move_as_ok());
|
||||
});
|
||||
td::actor::send_closure(root_db_, &RootDb::get_block_handle_external, N.block_id, false, std::move(P));
|
||||
td::actor::send_closure(root_db_, &RootDb::get_block_handle_external, block_id, false, std::move(P));
|
||||
}
|
||||
|
||||
void CellDbIn::gc_cont(BlockHandle handle) {
|
||||
|
|
@ -194,12 +214,30 @@ void CellDbIn::gc_cont(BlockHandle handle) {
|
|||
}
|
||||
|
||||
void CellDbIn::gc_cont2(BlockHandle handle) {
|
||||
td::PerfWarningTimer timer{"gccell", 0.1};
|
||||
enqueue([this, handle = std::move(handle)](td::Result<td::Unit> R) mutable {
|
||||
if (R.is_error()) {
|
||||
return;
|
||||
}
|
||||
td::Promise<td::Unit> promise = [timer = td::PerfWarningTimer{"gccell", 0.1}](td::Result<td::Unit>) {};
|
||||
auto FR = get_block(get_key_hash(handle->id()));
|
||||
FR.ensure();
|
||||
auto F = FR.move_as_ok();
|
||||
auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok();
|
||||
|
||||
auto FR = get_block(last_gc_);
|
||||
boc_->dec(cell);
|
||||
boc_->prepare_commit_async(async_executor, [SelfId = actor_id(this), promise = std::move(promise),
|
||||
block_id = handle->id()](td::Result<td::Unit> R) mutable {
|
||||
R.ensure();
|
||||
td::actor::send_closure(SelfId, &CellDbIn::gc_cont3, block_id, std::move(promise));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise) {
|
||||
auto key_hash = get_key_hash(block_id);
|
||||
auto FR = get_block(key_hash);
|
||||
FR.ensure();
|
||||
auto F = FR.move_as_ok();
|
||||
|
||||
auto PR = get_block(F.prev);
|
||||
PR.ensure();
|
||||
auto P = PR.move_as_ok();
|
||||
|
|
@ -214,14 +252,10 @@ void CellDbIn::gc_cont2(BlockHandle handle) {
|
|||
N.next = N.prev;
|
||||
}
|
||||
|
||||
auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok();
|
||||
|
||||
boc_->dec(cell);
|
||||
boc_->prepare_commit().ensure();
|
||||
vm::CellStorer stor{*cell_db_.get()};
|
||||
vm::CellStorer stor{*cell_db_};
|
||||
cell_db_->begin_write_batch().ensure();
|
||||
boc_->commit(stor).ensure();
|
||||
cell_db_->erase(get_key(last_gc_)).ensure();
|
||||
cell_db_->erase(get_key(key_hash)).ensure();
|
||||
set_block(F.prev, std::move(P));
|
||||
set_block(F.next, std::move(N));
|
||||
cell_db_->commit_write_batch().ensure();
|
||||
|
|
@ -230,16 +264,33 @@ void CellDbIn::gc_cont2(BlockHandle handle) {
|
|||
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot())).ensure();
|
||||
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
|
||||
|
||||
DCHECK(get_block(last_gc_).is_error());
|
||||
last_gc_ = F.next;
|
||||
DCHECK(get_block(key_hash).is_error());
|
||||
promise.set_result(td::Unit());
|
||||
release_db();
|
||||
}
|
||||
|
||||
void CellDbIn::enqueue(td::Promise<td::Unit> promise) {
|
||||
db_queue_.push(std::move(promise));
|
||||
process_event();
|
||||
}
|
||||
|
||||
void CellDbIn::release_db() {
|
||||
db_busy_ = false;
|
||||
process_event();
|
||||
}
|
||||
|
||||
void CellDbIn::process_event() {
|
||||
if (db_busy_ || db_queue_.empty()) {
|
||||
return;
|
||||
}
|
||||
db_busy_ = true;
|
||||
auto promise = std::move(db_queue_.front());
|
||||
db_queue_.pop();
|
||||
promise.set_result(td::Unit());
|
||||
}
|
||||
|
||||
void CellDbIn::skip_gc() {
|
||||
auto FR = get_block(last_gc_);
|
||||
FR.ensure();
|
||||
auto F = FR.move_as_ok();
|
||||
last_gc_ = F.next;
|
||||
alarm_timestamp() = td::Timestamp::in(0.01);
|
||||
alarm_timestamp() = td::Timestamp::in(1.0);
|
||||
}
|
||||
|
||||
std::string CellDbIn::get_key(KeyHash key_hash) {
|
||||
|
|
@ -296,7 +347,7 @@ void CellDb::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise
|
|||
} else {
|
||||
promise.set_result(R.move_as_ok());
|
||||
}
|
||||
});
|
||||
});
|
||||
boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue