external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-02-12 11:12:16 +00:00
Code Issues Releases Wiki Activity

Rollback celldb optimization (#658)

Browse source
This commit is contained in:
SpyCheese 2023-03-30 07:03:05 +00:00 committed by GitHub
parent 9be3701bc0
commit 5e0dadfff6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 21 additions and 285 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crypto/vm/db/CellHashTable.h
View file

@ -43,7 +43,7 @@ class CellHashTable {
template <class F>
void for_each(F &&f) {
for (auto &info : set_) {
f(const_cast<InfoT &>(info));
f(info);
}
}
template <class F>

14
crypto/vm/db/CellStorage.cpp
View file

@ -150,20 +150,6 @@ td::Result<CellLoader::LoadResult> CellLoader::load(td::Slice hash, bool need_da
return res;
}
td::Result<CellLoader::LoadResult> CellLoader::load_refcnt(td::Slice hash) {
LoadResult res;
std::string serialized;
TRY_RESULT(get_status, reader_->get(hash, serialized));
if (get_status != KeyValue::GetStatus::Ok) {
DCHECK(get_status == KeyValue::GetStatus::NotFound);
return res;
}
res.status = LoadResult::Ok;
td::TlParser parser(serialized);
td::parse(res.refcnt_, parser);
return res;
}
CellStorer::CellStorer(KeyValue &kv) : kv_(kv) {
}

1
crypto/vm/db/CellStorage.h
View file

@ -48,7 +48,6 @@ class CellLoader {
};
CellLoader(std::shared_ptr<KeyValueReader> reader);
td::Result<LoadResult> load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator);
td::Result<LoadResult> load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null
private:
std::shared_ptr<KeyValueReader> reader_;

174
crypto/vm/db/DynamicBagOfCellsDb.cpp
View file

@ -27,9 +27,6 @@
#include "td/utils/ThreadSafeCounter.h"
#include "vm/cellslice.h"
#include <queue>
#include "td/actor/actor.h"
#include "common/delay.h"
namespace vm {
namespace {
@ -558,177 +555,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
DynamicBocExtCellExtra{cell_db_reader_}));
return std::move(res);
}
struct PrepareCommitAsyncState {
size_t remaining_ = 0;
std::shared_ptr<AsyncExecutor> executor_;
td::Promise<td::Unit> promise_;
struct CellInfo2 {
CellInfo *info;
std::vector<CellInfo2 *> parents;
unsigned remaining_children = 0;
Cell::Hash key() const {
return info->key();
}
bool operator<(const CellInfo2 &other) const {
return key() < other.key();
}
friend bool operator<(const CellInfo2 &a, td::Slice b) {
return a.key().as_slice() < b;
}
friend bool operator<(td::Slice a, const CellInfo2 &b) {
return a < b.key().as_slice();
}
};
CellHashTable<CellInfo2> cells_;
};
std::unique_ptr<PrepareCommitAsyncState> pca_state_;
void prepare_commit_async(std::shared_ptr<AsyncExecutor> executor, td::Promise<td::Unit> promise) override {
if (pca_state_) {
promise.set_error(td::Status::Error("Other prepare_commit_async is not finished"));
return;
}
if (is_prepared_for_commit()) {
promise.set_result(td::Unit());
return;
}
pca_state_ = std::make_unique<PrepareCommitAsyncState>();
pca_state_->executor_ = std::move(executor);
pca_state_->promise_ = std::move(promise);
for (auto &new_cell : to_inc_) {
dfs_new_cells_in_db_async(new_cell);
}
pca_state_->cells_.for_each([&](PrepareCommitAsyncState::CellInfo2 &info) {
++pca_state_->remaining_;
if (info.remaining_children == 0) {
pca_load_from_db(&info);
}
});
if (pca_state_->remaining_ == 0) {
prepare_commit_async_cont();
}
}
void dfs_new_cells_in_db_async(const td::Ref<vm::Cell> &cell, PrepareCommitAsyncState::CellInfo2 *parent = nullptr) {
bool exists = true;
pca_state_->cells_.apply(cell->get_hash().as_slice(), [&](PrepareCommitAsyncState::CellInfo2 &info) {
if (info.info == nullptr) {
exists = false;
info.info = &get_cell_info(cell);
}
});
auto info = pca_state_->cells_.get_if_exists(cell->get_hash().as_slice());
if (parent) {
info->parents.push_back(parent);
++parent->remaining_children;
}
if (exists) {
return;
}
if (cell->is_loaded()) {
vm::CellSlice cs(vm::NoVm{}, cell);
for (unsigned i = 0; i < cs.size_refs(); i++) {
dfs_new_cells_in_db_async(cs.prefetch_ref(i), info);
}
}
}
void pca_load_from_db(PrepareCommitAsyncState::CellInfo2 *info) {
pca_state_->executor_->execute_async(
[db = this, info, executor = pca_state_->executor_, loader = *loader_]() mutable {
auto res = loader.load_refcnt(info->info->cell->get_hash().as_slice()).move_as_ok();
executor->execute_sync([db, info, res = std::move(res)]() {
db->pca_set_in_db(info, std::move(res));
});
});
}
void pca_set_in_db(PrepareCommitAsyncState::CellInfo2 *info, CellLoader::LoadResult result) {
info->info->sync_with_db = true;
if (result.status == CellLoader::LoadResult::Ok) {
info->info->in_db = true;
info->info->db_refcnt = result.refcnt();
} else {
info->info->in_db = false;
}
for (PrepareCommitAsyncState::CellInfo2 *parent_info : info->parents) {
if (parent_info->info->sync_with_db) {
continue;
}
if (!info->info->in_db) {
pca_set_in_db(parent_info, {});
} else if (--parent_info->remaining_children == 0) {
pca_load_from_db(parent_info);
}
}
if (--pca_state_->remaining_ == 0) {
prepare_commit_async_cont();
}
}
void prepare_commit_async_cont() {
for (auto &new_cell : to_inc_) {
auto &new_cell_info = get_cell_info(new_cell);
dfs_new_cells(new_cell_info);
}
CHECK(pca_state_->remaining_ == 0);
for (auto &old_cell : to_dec_) {
auto &old_cell_info = get_cell_info(old_cell);
dfs_old_cells_async(old_cell_info);
}
if (pca_state_->remaining_ == 0) {
prepare_commit_async_cont2();
}
}
void dfs_old_cells_async(CellInfo &info) {
if (!info.was) {
info.was = true;
visited_.push_back(&info);
if (!info.sync_with_db) {
++pca_state_->remaining_;
load_cell_async(
info.cell->get_hash().as_slice(), pca_state_->executor_,
[executor = pca_state_->executor_, db = this, info = &info](td::Result<td::Ref<vm::DataCell>> R) {
R.ensure();
executor->execute_sync([db, info]() {
CHECK(info->sync_with_db);
db->dfs_old_cells_async(*info);
if (--db->pca_state_->remaining_ == 0) {
db->prepare_commit_async_cont2();
}
});
});
return;
}
}
info.refcnt_diff--;
if (!info.sync_with_db) {
return;
}
auto new_refcnt = info.refcnt_diff + info.db_refcnt;
CHECK(new_refcnt >= 0);
if (new_refcnt != 0) {
return;
}
for_each(info, [this](auto &child_info) { dfs_old_cells_async(child_info); });
}
void prepare_commit_async_cont2() {
save_diff_prepare();
to_inc_.clear();
to_dec_.clear();
pca_state_->promise_.set_result(td::Unit());
pca_state_ = {};
}
};
} // namespace

1
crypto/vm/db/DynamicBagOfCellsDb.h
View file

@ -75,7 +75,6 @@ class DynamicBagOfCellsDb {
virtual void load_cell_async(td::Slice hash, std::shared_ptr<AsyncExecutor> executor,
td::Promise<Ref<DataCell>> promise) = 0;
virtual void prepare_commit_async(std::shared_ptr<AsyncExecutor> executor, td::Promise<td::Unit> promise) = 0;
};
} // namespace vm

103
validator/db/celldb.cpp
View file

@ -86,42 +86,19 @@ void CellDbIn::start_up() {
}
void CellDbIn::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
enqueue([this, hash, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
return;
}
promise.set_result(boc_->load_cell(hash.as_slice()));
release_db();
});
boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise));
}
void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise) {
enqueue([this, block_id, cell = std::move(cell), promise = std::move(promise)](td::Result<td::Unit> R0) mutable {
if (R0.is_error()) {
return;
}
promise = promise.wrap([timer = td::PerfWarningTimer{"storecell", 0.1}](td::Ref<vm::DataCell> &&r) { return r; });
auto key_hash = get_key_hash(block_id);
auto R = get_block(key_hash);
// duplicate
if (R.is_ok()) {
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
release_db();
return;
}
boc_->inc(cell);
boc_->prepare_commit_async(
async_executor, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::Unit> R) mutable {
R.ensure();
td::actor::send_closure(SelfId, &CellDbIn::store_cell_cont, block_id, cell, std::move(promise));
});
});
}
void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell,
td::Promise<td::Ref<vm::DataCell>> promise) {
td::PerfWarningTimer timer{"storecell", 0.1};
auto key_hash = get_key_hash(block_id);
auto R = get_block(key_hash);
// duplicate
if (R.is_ok()) {
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
return;
}
auto empty = get_empty_key_hash();
auto ER = get_block(empty);
ER.ensure();
@ -142,7 +119,9 @@ void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell,
P.prev = key_hash;
}
vm::CellStorer stor{*cell_db_};
boc_->inc(cell);
boc_->prepare_commit().ensure();
vm::CellStorer stor{*cell_db_.get()};
cell_db_->begin_write_batch().ensure();
boc_->commit(stor).ensure();
set_block(empty, std::move(E));
@ -154,17 +133,10 @@ void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell,
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
release_db();
}
void CellDbIn::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) {
enqueue([this, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
return;
}
promise.set_result(boc_->get_cell_db_reader());
release_db();
});
promise.set_result(boc_->get_cell_db_reader());
}
void CellDbIn::alarm() {
@ -214,30 +186,13 @@ void CellDbIn::gc_cont(BlockHandle handle) {
}
void CellDbIn::gc_cont2(BlockHandle handle) {
enqueue([this, handle = std::move(handle)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
return;
}
td::Promise<td::Unit> promise = [timer = td::PerfWarningTimer{"gccell", 0.1}](td::Result<td::Unit>) {};
auto FR = get_block(get_key_hash(handle->id()));
FR.ensure();
auto F = FR.move_as_ok();
auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok();
td::PerfWarningTimer timer{"gccell", 0.1};
boc_->dec(cell);
boc_->prepare_commit_async(async_executor, [SelfId = actor_id(this), promise = std::move(promise),
block_id = handle->id()](td::Result<td::Unit> R) mutable {
R.ensure();
td::actor::send_closure(SelfId, &CellDbIn::gc_cont3, block_id, std::move(promise));
});
});
}
void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise) {
auto key_hash = get_key_hash(block_id);
auto key_hash = get_key_hash(handle->id());
auto FR = get_block(key_hash);
FR.ensure();
auto F = FR.move_as_ok();
auto PR = get_block(F.prev);
PR.ensure();
auto P = PR.move_as_ok();
@ -252,6 +207,10 @@ void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise) {
N.next = N.prev;
}
auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok();
boc_->dec(cell);
boc_->prepare_commit().ensure();
vm::CellStorer stor{*cell_db_};
cell_db_->begin_write_batch().ensure();
boc_->commit(stor).ensure();
@ -265,28 +224,6 @@ void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise) {
td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
DCHECK(get_block(key_hash).is_error());
promise.set_result(td::Unit());
release_db();
}
void CellDbIn::enqueue(td::Promise<td::Unit> promise) {
db_queue_.push(std::move(promise));
process_event();
}
void CellDbIn::release_db() {
db_busy_ = false;
process_event();
}
void CellDbIn::process_event() {
if (db_busy_ || db_queue_.empty()) {
return;
}
db_busy_ = true;
auto promise = std::move(db_queue_.front());
db_queue_.pop();
promise.set_result(td::Unit());
}
void CellDbIn::skip_gc() {

11
validator/db/celldb.hpp
View file

@ -25,7 +25,6 @@
#include "ton/ton-types.h"
#include "interfaces/block-handle.h"
#include "auto/tl/ton_api.h"
#include <queue>
namespace ton {
@ -88,11 +87,8 @@ class CellDbIn : public CellDbBase {
void gc(BlockIdExt block_id);
void gc_cont(BlockHandle handle);
void gc_cont2(BlockHandle handle);
void gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise);
void skip_gc();
void store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise);
td::actor::ActorId<RootDb> root_db_;
td::actor::ActorId<CellDb> parent_;
@ -100,13 +96,6 @@ class CellDbIn : public CellDbBase {
std::unique_ptr<vm::DynamicBagOfCellsDb> boc_;
std::shared_ptr<vm::KeyValue> cell_db_;
std::queue<td::Promise<td::Unit>> db_queue_;
bool db_busy_ = false;
void enqueue(td::Promise<td::Unit> promise);
void release_db();
void process_event();
};
class CellDb : public CellDbBase {