Add option --celldb-cache-size (#988)

Co-authored-by: SpyCheese <mikle98@yandex.ru>

tddb/td/db/RocksDb.cpp

@@ -59,38 +59,38 @@ RocksDb RocksDb::clone() const {
   return RocksDb{db_, statistics_};
 }
 
-Result<RocksDb> RocksDb::open(std::string path, std::shared_ptr<rocksdb::Statistics> statistics) {
+Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
   rocksdb::OptimisticTransactionDB *db;
   {
-    rocksdb::Options options;
+    rocksdb::Options db_options;
 
-    static auto cache = rocksdb::NewLRUCache(1 << 30);
+    static auto cache = rocksdb::NewLRUCache(options.block_cache_size);
 
     rocksdb::BlockBasedTableOptions table_options;
     table_options.block_cache = cache;
-    options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options));
+    db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options));
 
-    options.manual_wal_flush = true;
+    db_options.manual_wal_flush = true;
-    options.create_if_missing = true;
+    db_options.create_if_missing = true;
-    options.max_background_compactions = 4;
+    db_options.max_background_compactions = 4;
-    options.max_background_flushes = 2;
+    db_options.max_background_flushes = 2;
-    options.bytes_per_sync = 1 << 20;
+    db_options.bytes_per_sync = 1 << 20;
-    options.writable_file_max_buffer_size = 2 << 14;
+    db_options.writable_file_max_buffer_size = 2 << 14;
-    options.statistics = statistics;
+    db_options.statistics = options.statistics;
     rocksdb::OptimisticTransactionDBOptions occ_options;
     occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial;
-    rocksdb::ColumnFamilyOptions cf_options(options);
+    rocksdb::ColumnFamilyOptions cf_options(db_options);
     std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
     column_families.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, cf_options));
     std::vector<rocksdb::ColumnFamilyHandle *> handles;
-    TRY_STATUS(from_rocksdb(
+    TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(db_options, occ_options, std::move(path),
-        rocksdb::OptimisticTransactionDB::Open(options, occ_options, std::move(path), column_families, &handles, &db)));
+                                                                   column_families, &handles, &db)));
     CHECK(handles.size() == 1);
     // i can delete the handle since DBImpl is always holding a reference to
     // default column family
     delete handles[0];
   }
-  return RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB>(db), std::move(statistics));
+  return RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB>(db), std::move(options.statistics));
 }
 
 std::shared_ptr<rocksdb::Statistics> RocksDb::create_statistics() {

tddb/td/db/RocksDb.h

@@ -34,11 +34,17 @@ class Statistics;
 }  // namespace rocksdb
 
 namespace td {
+
+struct RocksDbOptions {
+  std::shared_ptr<rocksdb::Statistics> statistics = nullptr;
+  uint64 block_cache_size = 1 << 30;
+};
+
 class RocksDb : public KeyValue {
  public:
   static Status destroy(Slice path);
   RocksDb clone() const;
-  static Result<RocksDb> open(std::string path, std::shared_ptr<rocksdb::Statistics> statistics = nullptr);
+  static Result<RocksDb> open(std::string path, RocksDbOptions options = {});
 
   Result<GetStatus> get(Slice key, std::string &value) override;
   Status set(Slice key, Slice value) override;

validator-engine/validator-engine.cpp

@@ -1369,6 +1369,9 @@ td::Status ValidatorEngine::load_global_config() {
   validator_options_.write().set_archive_preload_period(archive_preload_period_);
   validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_);
   validator_options_.write().set_nonfinal_ls_queries_enabled(nonfinal_ls_queries_enabled_);
+  if (celldb_cache_size_) {
+    validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value());
+  }
 
   std::vector<ton::BlockIdExt> h;
   for (auto &x : conf.validator_->hardforks_) {
@@ -3967,6 +3970,16 @@ int main(int argc, char *argv[]) {
   p.add_option('\0', "nonfinal-ls", "enable special LS queries to non-finalized blocks", [&]() {
     acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); });
   });
+  p.add_checked_option(
+      '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)",
+      [&](td::Slice s) -> td::Status {
+        TRY_RESULT(v, td::to_integer_safe<td::uint64>(s));
+        if (v == 0) {
+          return td::Status::Error("celldb-cache-size should be positive");
+        }
+        acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); });
+        return td::Status::OK();
+      });
   auto S = p.run(argc, argv);
   if (S.is_error()) {
     LOG(ERROR) << "failed to parse options: " << S.move_as_error();

validator-engine/validator-engine.hpp

@@ -209,6 +209,7 @@ class ValidatorEngine : public td::actor::Actor {
   double archive_preload_period_ = 0.0;
   bool disable_rocksdb_stats_ = false;
   bool nonfinal_ls_queries_enabled_ = false;
+  td::optional<td::uint64> celldb_cache_size_;
   bool read_config_ = false;
   bool started_keyring_ = false;
   bool started_ = false;
@@ -281,6 +282,9 @@ class ValidatorEngine : public td::actor::Actor {
   void set_nonfinal_ls_queries_enabled() {
     nonfinal_ls_queries_enabled_ = true;
   }
+  void set_celldb_cache_size(td::uint64 value) {
+    celldb_cache_size_ = value;
+  }
   void start_up() override;
   ValidatorEngine() {
   }

validator/db/archive-manager.cpp

@@ -832,7 +832,10 @@ void ArchiveManager::start_up() {
   if (!opts_->get_disable_rocksdb_stats()) {
     statistics_.init();
   }
-  index_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_root_ + "/files/globalindex", statistics_.rocksdb_statistics).move_as_ok());
+  td::RocksDbOptions db_options;
+  db_options.statistics = statistics_.rocksdb_statistics;
+  index_ = std::make_shared<td::RocksDb>(
+      td::RocksDb::open(db_root_ + "/files/globalindex", std::move(db_options)).move_as_ok());
   std::string value;
   auto v = index_->get(create_serialize_tl_object<ton_api::db_files_index_key>().as_slice(), value);
   v.ensure();

validator/db/archive-slice.cpp

@@ -554,7 +554,9 @@ void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::
 void ArchiveSlice::before_query() {
   if (status_ == st_closed) {
     LOG(DEBUG) << "Opening archive slice " << db_path_;
-    kv_ = std::make_unique<td::RocksDb>(td::RocksDb::open(db_path_, statistics_.rocksdb_statistics).move_as_ok());
+    td::RocksDbOptions db_options;
+    db_options.statistics = statistics_.rocksdb_statistics;
+    kv_ = std::make_unique<td::RocksDb>(td::RocksDb::open(db_path_, std::move(db_options)).move_as_ok());
     std::string value;
     auto R2 = kv_->get("status", value);
     R2.ensure();

validator/db/celldb.cpp

@@ -88,7 +88,13 @@ void CellDbIn::start_up() {
     statistics_ = td::RocksDb::create_statistics();
     statistics_flush_at_ = td::Timestamp::in(60.0);
   }
-  cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_, statistics_).move_as_ok());
+  td::RocksDbOptions db_options;
+  db_options.statistics = statistics_;
+  if (opts_->get_celldb_cache_size()) {
+    db_options.block_cache_size = opts_->get_celldb_cache_size().value();
+    LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(db_options.block_cache_size);
+  }
+  cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_, std::move(db_options)).move_as_ok());
 
 
   boc_ = vm::DynamicBagOfCellsDb::create();

validator/validator-options.hpp

@@ -129,6 +129,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   bool nonfinal_ls_queries_enabled() const override {
     return nonfinal_ls_queries_enabled_;
   }
+  td::optional<td::uint64> get_celldb_cache_size() const override {
+    return celldb_cache_size_;
+  }
 
   void set_zero_block_id(BlockIdExt block_id) override {
     zero_block_id_ = block_id;
@@ -197,6 +200,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   void set_nonfinal_ls_queries_enabled(bool value) override {
     nonfinal_ls_queries_enabled_ = value;
   }
+  void set_celldb_cache_size(td::uint64 value) override {
+    celldb_cache_size_ = value;
+  }
 
   ValidatorManagerOptionsImpl *make_copy() const override {
     return new ValidatorManagerOptionsImpl(*this);
@@ -244,6 +250,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   double archive_preload_period_ = 0.0;
   bool disable_rocksdb_stats_;
   bool nonfinal_ls_queries_enabled_ = false;
+  td::optional<td::uint64> celldb_cache_size_;
 };
 
 }  // namespace validator

validator/validator.h

@@ -86,6 +86,7 @@ struct ValidatorManagerOptions : public td::CntObject {
   virtual double get_archive_preload_period() const = 0;
   virtual bool get_disable_rocksdb_stats() const = 0;
   virtual bool nonfinal_ls_queries_enabled() const = 0;
+  virtual td::optional<td::uint64> get_celldb_cache_size() const = 0;
 
   virtual void set_zero_block_id(BlockIdExt block_id) = 0;
   virtual void set_init_block_id(BlockIdExt block_id) = 0;
@@ -110,6 +111,7 @@ struct ValidatorManagerOptions : public td::CntObject {
   virtual void set_archive_preload_period(double value) = 0;
   virtual void set_disable_rocksdb_stats(bool value) = 0;
   virtual void set_nonfinal_ls_queries_enabled(bool value) = 0;
+  virtual void set_celldb_cache_size(td::uint64 value) = 0;
 
   static td::Ref<ValidatorManagerOptions> create(
       BlockIdExt zero_block_id, BlockIdExt init_block_id,