external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

celldb: add test for load nonexisting cell, test thread safeness of CellUsageTree, fixes

Browse source
This commit is contained in:
birydrad 2025-01-15 14:25:25 +00:00
parent c863c42ed1
commit 597f3b7020
7 changed files with 130 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

35
crypto/test/test-db.cpp
View file

@ -50,6 +50,7 @@
#include <set>
#include <map>
#include <thread>
#include <barrier>
#include <openssl/sha.h>
@ -1312,7 +1313,8 @@ void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) {
// V2 - one thread
run({.async_executor = executor,
.kv_options = kv_options,
.options = DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}});
.options =
DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}});
// InMemory
for (auto use_arena : {false, true}) {
@ -1353,6 +1355,8 @@ DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) {
if (rnd() % 10 == 0) {
reload_db();
}
db.dboc->load_cell(vm::CellHash{}.as_slice()).ensure_error();
db.reset_loader();
Ref<Cell> old_root;
if (!old_root_hash.empty()) {
@ -2901,6 +2905,35 @@ TEST(TonDb, BocRespectsUsageCell) {
ASSERT_STREQ(serialization, serialization_of_virtualized_cell);
}
TEST(UsageTree, ThreadSafe) {
size_t test_n = 100;
td::Random::Xorshift128plus rnd(123);
for (size_t test_i = 0; test_i < test_n; test_i++) {
auto cell = vm::gen_random_cell(rnd.fast(2, 100), rnd, false);
auto usage_tree = std::make_shared<vm::CellUsageTree>();
auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr());
std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4
auto barrier = std::barrier{threads_n};
std::vector<std::thread> threads;
std::vector<vm::CellExplorer::Exploration> explorations(threads_n);
for (std::ptrdiff_t i = 0; i < threads_n; i++) {
threads.emplace_back([&, i = i]() {
barrier.arrive_and_wait();
explorations[i] = vm::CellExplorer::random_explore(usage_cell, rnd);
});
}
for (auto &thread : threads) {
thread.join();
}
auto proof = vm::MerkleProof::generate(cell, usage_tree.get());
auto virtualized_proof = vm::MerkleProof::virtualize(proof, 1);
for (auto &exploration : explorations) {
auto new_exploration = vm::CellExplorer::explore(virtualized_proof, exploration.ops);
ASSERT_EQ(exploration.log, new_exploration.log);
}
}
}
/*
vm::DynamicBagOfCellsDb::Stats test_dynamic_boc_respects_usage_cell(vm::BocOptions options) {
td::Random::Xorshift128plus rnd(options.seed);

8
crypto/vm/db/DynamicBagOfCellsDb.cpp
View file

@ -107,15 +107,21 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
td::Result<Ref<Cell>> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override {
return get_cell_info_lazy(level_mask, hash, depth).cell;
}
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const override {
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const override {
std::vector<std::pair<std::string, std::string>> result;
auto s = loader_->key_value_reader().for_each_in_range("desc", "desd",
[&](const td::Slice &key, const td::Slice &value) {
if (result.size() >= max_count) {
return td::Status::Error("COUNT_LIMIT");
}
if (td::begins_with(key, "desc") && key.size() != 32) {
result.emplace_back(key.str(), value.str());
}
return td::Status::OK();
});
if (s.message() == "COUNT_LIMIT") {
s = td::Status::OK();
}
TRY_STATUS(std::move(s));
return result;
}

2
crypto/vm/db/DynamicBagOfCellsDb.h
View file

@ -51,7 +51,7 @@ class DynamicBagOfCellsDb {
public:
virtual ~DynamicBagOfCellsDb() = default;
virtual td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const = 0;
virtual td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const = 0;
virtual td::Result<td::KeyValue::GetStatus> meta_get(td::Slice key, std::string &value) = 0;
virtual td::Status meta_set(td::Slice key, td::Slice value) = 0;
virtual td::Status meta_erase(td::Slice key) = 0;

8
crypto/vm/db/DynamicBagOfCellsDbV2.cpp
View file

@ -765,16 +765,22 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb {
}
}
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const override {
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const override {
CHECK(meta_db_fixup_.empty());
std::vector<std::pair<std::string, std::string>> result;
auto s = cell_db_reader_->key_value_reader().for_each_in_range(
"desc", "desd", [&](const td::Slice &key, const td::Slice &value) {
if (result.size() >= max_count) {
return td::Status::Error("COUNT_LIMIT");
}
if (td::begins_with(key, "desc") && key.size() != 32) {
result.emplace_back(key.str(), value.str());
}
return td::Status::OK();
});
if (s.message() == "COUNT_LIMIT") {
s = td::Status::OK();
}
TRY_STATUS(std::move(s));
return result;
}

15
crypto/vm/db/InMemoryBagOfCellsDb.cpp
View file

@ -774,8 +774,15 @@ class MetaStorage {
CHECK(p.first.size() != CellTraits::hash_bytes);
}
}
std::vector<std::pair<std::string, std::string>> meta_get_all() const {
return td::transform(meta_, [](const auto &p) { return std::make_pair(p.first, p.second); });
std::vector<std::pair<std::string, std::string>> meta_get_all(size_t max_count) const {
std::vector<std::pair<std::string, std::string>> res;
for (const auto &[k, v] : meta_) {
if (res.size() >= max_count) {
break;
}
res.emplace_back(k, v);
}
return res;
}
KeyValue::GetStatus meta_get(td::Slice key, std::string &value) const {
auto lock = local_access_.lock();
@ -814,8 +821,8 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb {
: storage_(std::move(storage)), meta_storage_(std::move(meta_storage)) {
}
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const override {
return meta_storage_->meta_get_all();
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const override {
return meta_storage_->meta_get_all(max_count);
}
td::Result<KeyValue::GetStatus> meta_get(td::Slice key, std::string &value) override {
CHECK(key.size() != CellTraits::hash_bytes);

122
validator/db/celldb.cpp
View file

@ -111,6 +111,66 @@ struct MergeOperatorAddCellRefcnt : public rocksdb::MergeOperator {
}
};
void CellDbIn::validate_meta() {
LOG(INFO) << "Validating metadata\n";
size_t max_meta_keys_loaded = opts_->get_celldb_in_memory() ? std::numeric_limits<std::size_t>::max() : 10000;
auto meta = boc_->meta_get_all(max_meta_keys_loaded).move_as_ok();
bool partial_check = meta.size() == max_meta_keys_loaded;
if (partial_check) {
LOG(ERROR) << "Too much metadata in the database, do only partial check";
}
size_t missing_roots = 0;
size_t unknown_roots = 0;
std::set<vm::CellHash> root_hashes;
for (auto [k, v] : meta) {
if (k == "desczero") {
continue;
}
auto obj = fetch_tl_object<ton_api::db_celldb_value>(td::BufferSlice{v}, true);
obj.ensure();
auto entry = DbEntry{obj.move_as_ok()};
root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice()));
auto cell = boc_->load_cell(entry.root_hash.as_slice());
missing_roots += cell.is_error();
LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error();
}
// load_known_roots is only supported by InMemory database, so it is ok to check all known roots here
auto known_roots = boc_->load_known_roots().move_as_ok();
for (auto& root : known_roots) {
block::gen::ShardStateUnsplit::Record info;
block::gen::OutMsgQueueInfo::Record qinfo;
block::ShardId shard;
if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) &&
tlb::unpack_cell(info.out_msg_queue_info, qinfo))) {
LOG(FATAL) << "cannot create ShardDescr from a root in celldb";
}
if (!partial_check && !root_hashes.contains(root->get_hash())) {
unknown_roots++;
LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no;
constexpr bool delete_unknown_roots = false;
if (delete_unknown_roots) {
vm::CellStorer stor{*cell_db_};
cell_db_->begin_write_batch().ensure();
boc_->dec(root);
boc_->commit(stor).ensure();
cell_db_->commit_write_batch().ensure();
if (!opts_->get_celldb_in_memory()) {
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
}
LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED";
}
}
}
LOG_IF(ERROR, missing_roots != 0) << "Missing root hashes: " << missing_roots;
LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
LOG_IF(FATAL, missing_roots != 0) << "Missing root hashes: " << missing_roots;
LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
LOG(INFO) << "Validating metadata: OK\n";
}
void CellDbIn::start_up() {
on_load_callback_ = [actor = std::make_shared<td::actor::ActorOwn<MigrationProxy>>(
td::actor::create_actor<MigrationProxy>("celldbmigration", actor_id(this))),
@ -141,11 +201,11 @@ void CellDbIn::start_up() {
std::optional<vm::DynamicBagOfCellsDb::CreateV2Options> boc_v2_options;
if (opts_->get_celldb_v2()) {
boc_v2_options =
vm::DynamicBagOfCellsDb::CreateV2Options{.extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u),
.executor = {},
.cache_ttl_max = 2000,
.cache_size_max = 1000000};
boc_v2_options = vm::DynamicBagOfCellsDb::CreateV2Options{
.extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u),
.executor = {},
.cache_ttl_max = 2000,
.cache_size_max = 1000000};
size_t min_rocksdb_cache = std::max(size_t{1} << 30, boc_v2_options->cache_size_max * 5000);
if (!o_celldb_cache_size || o_celldb_cache_size.value() < min_rocksdb_cache) {
LOG(WARNING) << "Increase CellDb block cache size to " << td::format::as_size(min_rocksdb_cache) << " from "
@ -208,54 +268,7 @@ void CellDbIn::start_up() {
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
}
auto meta = boc_->meta_get_all().move_as_ok();
size_t missing_roots = 0;
size_t unknown_roots = 0;
std::set<vm::CellHash> root_hashes;
for (auto [k, v] : meta) {
if (k == "desczero") {
continue;
}
auto obj = fetch_tl_object<ton_api::db_celldb_value>(td::BufferSlice{v}, true);
obj.ensure();
auto entry = DbEntry{obj.move_as_ok()};
root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice()));
auto cell = boc_->load_cell(entry.root_hash.as_slice());
missing_roots += cell.is_error();
LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error();
}
auto known_roots = boc_->load_known_roots().move_as_ok();
for (auto& root : known_roots) {
block::gen::ShardStateUnsplit::Record info;
block::gen::OutMsgQueueInfo::Record qinfo;
block::ShardId shard;
if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) &&
tlb::unpack_cell(info.out_msg_queue_info, qinfo))) {
LOG(FATAL) << "cannot create ShardDescr from a root in celldb";
}
if (!root_hashes.contains(root->get_hash())) {
unknown_roots++;
LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no;
constexpr bool delete_unknown_roots = false;
if (delete_unknown_roots) {
vm::CellStorer stor{*cell_db_};
cell_db_->begin_write_batch().ensure();
boc_->dec(root);
boc_->commit(stor).ensure();
cell_db_->commit_write_batch().ensure();
if (!opts_->get_celldb_in_memory()) {
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
}
LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED";
}
}
}
LOG_IF(ERROR, missing_roots != 1) << "Missing root hashes: " << missing_roots;
LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
LOG_IF(FATAL, missing_roots > 1) << "Missing root hashes: " << missing_roots;
LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
validate_meta();
alarm_timestamp() = td::Timestamp::in(10.0);
@ -267,6 +280,9 @@ void CellDbIn::start_up() {
set_block(empty, std::move(e));
boc_->commit(stor);
cell_db_->commit_write_batch().ensure();
if (!opts_->get_celldb_in_memory()) {
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
}
}
if (opts_->get_celldb_v2() || opts_->get_celldb_in_memory()) {

1
validator/db/celldb.hpp
View file

@ -74,6 +74,7 @@ class CellDbIn : public CellDbBase {
CellDbIn(td::actor::ActorId<RootDb> root_db, td::actor::ActorId<CellDb> parent, std::string path,
td::Ref<ValidatorManagerOptions> opts);
void validate_meta();
void start_up() override;
void alarm() override;