external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Add --celldb-direct-io and --celldb-preload-all (#993)

Browse source 
Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK 2024-05-14 16:05:29 +03:00 • committed by GitHub
parent 816dd9cf2d
commit 561f342639
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 108 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

4
tddb/td/db/KeyValue.h
View file

@ -19,6 +19,7 @@
#pragma once
#include "td/utils/Status.h"
#include "td/utils/logging.h"
#include <functional>
namespace td {
class KeyValueReader {
public:
@ -27,6 +28,9 @@ class KeyValueReader {
virtual Result<GetStatus> get(Slice key, std::string &value) = 0;
virtual Result<size_t> count(Slice prefix) = 0;
virtual Status for_each(std::function<Status(Slice, Slice)> f) {
return Status::Error("for_each is not supported");
}
};
class PrefixedKeyValueReader : public KeyValueReader {

47
tddb/td/db/RocksDb.cpp
View file

@ -56,7 +56,7 @@ RocksDb::~RocksDb() {
}
RocksDb RocksDb::clone() const {
return RocksDb{db_, statistics_};
return RocksDb{db_, options_};
}
Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
@ -64,16 +64,16 @@ Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
{
rocksdb::Options db_options;
static auto cache = rocksdb::NewLRUCache(1 << 30);
static auto default_cache = rocksdb::NewLRUCache(1 << 30);
if (options.block_cache == nullptr) {
options.block_cache = default_cache;
}
rocksdb::BlockBasedTableOptions table_options;
if (options.block_cache_size) {
table_options.block_cache = rocksdb::NewLRUCache(options.block_cache_size.value());
} else {
table_options.block_cache = cache;
}
table_options.block_cache = options.block_cache;
db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options));
db_options.use_direct_reads = options.use_direct_reads;
db_options.manual_wal_flush = true;
db_options.create_if_missing = true;
db_options.max_background_compactions = 4;
@ -94,7 +94,7 @@ Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
// default column family
delete handles[0];
}
return RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB>(db), std::move(options.statistics));
return RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB>(db), std::move(options));
}
std::shared_ptr<rocksdb::Statistics> RocksDb::create_statistics() {
@ -109,6 +109,10 @@ void RocksDb::reset_statistics(const std::shared_ptr<rocksdb::Statistics> statis
statistics->Reset();
}
std::shared_ptr<rocksdb::Cache> RocksDb::create_cache(size_t capacity) {
return rocksdb::NewLRUCache(capacity);
}
std::unique_ptr<KeyValueReader> RocksDb::snapshot() {
auto res = std::make_unique<RocksDb>(clone());
res->begin_snapshot().ensure();
@ -120,7 +124,6 @@ std::string RocksDb::stats() const {
db_->GetProperty("rocksdb.stats", &out);
//db_->GetProperty("rocksdb.cur-size-all-mem-tables", &out);
return out;
return statistics_->ToString();
}
Result<RocksDb::GetStatus> RocksDb::get(Slice key, std::string &value) {
@ -187,6 +190,28 @@ Result<size_t> RocksDb::count(Slice prefix) {
return res;
}
Status RocksDb::for_each(std::function<Status(Slice, Slice)> f) {
rocksdb::ReadOptions options;
options.snapshot = snapshot_.get();
std::unique_ptr<rocksdb::Iterator> iterator;
if (snapshot_ || !transaction_) {
iterator.reset(db_->NewIterator(options));
} else {
iterator.reset(transaction_->GetIterator(options));
}
iterator->SeekToFirst();
for (; iterator->Valid(); iterator->Next()) {
auto key = from_rocksdb(iterator->key());
auto value = from_rocksdb(iterator->value());
TRY_STATUS(f(key, value));
}
if (!iterator->status().ok()) {
return from_rocksdb(iterator->status());
}
return Status::OK();
}
Status RocksDb::begin_write_batch() {
CHECK(!transaction_);
write_batch_ = std::make_unique<rocksdb::WriteBatch>();
@ -243,7 +268,7 @@ Status RocksDb::end_snapshot() {
return td::Status::OK();
}
RocksDb::RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db, std::shared_ptr<rocksdb::Statistics> statistics)
: db_(std::move(db)), statistics_(std::move(statistics)) {
RocksDb::RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db, RocksDbOptions options)
: db_(std::move(db)), options_(options) {
}
} // namespace td

12
tddb/td/db/RocksDb.h
View file

@ -27,6 +27,7 @@
#include "td/utils/optional.h"
namespace rocksdb {
class Cache;
class OptimisticTransactionDB;
class Transaction;
class WriteBatch;
@ -38,7 +39,8 @@ namespace td {
struct RocksDbOptions {
std::shared_ptr<rocksdb::Statistics> statistics = nullptr;
optional<uint64> block_cache_size; // Default - one 1G cache for all RocksDb
std::shared_ptr<rocksdb::Cache> block_cache; // Default - one 1G cache for all RocksDb
bool use_direct_reads = false;
};
class RocksDb : public KeyValue {
@ -51,6 +53,7 @@ class RocksDb : public KeyValue {
Status set(Slice key, Slice value) override;
Status erase(Slice key) override;
Result<size_t> count(Slice prefix) override;
Status for_each(std::function<Status(Slice, Slice)> f) override;
Status begin_write_batch() override;
Status commit_write_batch() override;
@ -71,6 +74,8 @@ class RocksDb : public KeyValue {
static std::string statistics_to_string(const std::shared_ptr<rocksdb::Statistics> statistics);
static void reset_statistics(const std::shared_ptr<rocksdb::Statistics> statistics);
static std::shared_ptr<rocksdb::Cache> create_cache(size_t capacity);
RocksDb(RocksDb &&);
RocksDb &operator=(RocksDb &&);
~RocksDb();
@ -81,7 +86,7 @@ class RocksDb : public KeyValue {
private:
std::shared_ptr<rocksdb::OptimisticTransactionDB> db_;
std::shared_ptr<rocksdb::Statistics> statistics_;
RocksDbOptions options_;
std::unique_ptr<rocksdb::Transaction> transaction_;
std::unique_ptr<rocksdb::WriteBatch> write_batch_;
@ -94,7 +99,6 @@ class RocksDb : public KeyValue {
};
std::unique_ptr<const rocksdb::Snapshot, UnreachableDeleter> snapshot_;
explicit RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db,
std::shared_ptr<rocksdb::Statistics> statistics);
explicit RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db, RocksDbOptions options);
};
} // namespace td