celldb in-memory mode, stats for actors, perf counters, minor fix in rldp2 (#1164)

* getactorstats query for validator-engine-console

* celldb in-memory mode (--celldb-in-memory option)

* rldp2: bugfix - do not estimate speed while nothing is sent

* add simple ed25519 benchmark

* fix compilation errors of different platforms and move to c++20

* fix some warnings

* turn on TON_USE_ABSEIL for glibc 2.27 nix build

---------

Co-authored-by: birydrad <>

tddb/CMakeLists.txt

@@ -59,7 +59,7 @@ target_link_libraries(tddb PUBLIC tdutils tdactor tddb_utils)
 if (TDDB_USE_ROCKSDB)
   target_sources(tddb PRIVATE ${TDDB_ROCKSDB_SOURCE})
   target_compile_definitions(tddb PUBLIC -DTDDB_USE_ROCKSDB)
-  target_link_libraries(tddb PRIVATE rocksdb)
+  target_link_libraries(tddb PUBLIC rocksdb)
   target_include_directories(tddb PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../third-party/rocksdb/include>)
 endif()
 

tddb/td/db/KeyValue.h

@@ -32,6 +32,9 @@ class KeyValueReader {
   virtual Status for_each(std::function<Status(Slice, Slice)> f) {
     return Status::Error("for_each is not supported");
   }
+  virtual Status for_each_in_range (Slice begin, Slice end, std::function<Status(Slice, Slice)> f) {
+    return td::Status::Error("foreach_range is not supported");
+  }
 };
 
 class PrefixedKeyValueReader : public KeyValueReader {

tddb/td/db/MemoryKeyValue.cpp

@@ -29,6 +29,24 @@ Result<MemoryKeyValue::GetStatus> MemoryKeyValue::get(Slice key, std::string &va
   value = it->second;
   return GetStatus::Ok;
 }
+
+Status MemoryKeyValue::for_each(std::function<Status(Slice, Slice)> f) {
+  for (auto &it : map_) {
+    TRY_STATUS(f(it.first, it.second));
+  }
+  return Status::OK();
+}
+
+Status MemoryKeyValue::for_each_in_range(Slice begin, Slice end, std::function<Status(Slice, Slice)> f) {
+  for (auto it = map_.lower_bound(begin); it != map_.end(); it++) {
+    if (it->first < end) {
+      TRY_STATUS(f(it->first, it->second));
+    } else {
+      break;
+    }
+  }
+  return Status::OK();
+}
 Status MemoryKeyValue::set(Slice key, Slice value) {
   map_[key.str()] = value.str();
   return Status::OK();

tddb/td/db/MemoryKeyValue.h

@@ -25,6 +25,8 @@ namespace td {
 class MemoryKeyValue : public KeyValue {
  public:
   Result<GetStatus> get(Slice key, std::string &value) override;
+  Status for_each(std::function<Status(Slice, Slice)> f) override;
+  Status for_each_in_range(Slice begin, Slice end, std::function<Status(Slice, Slice)> f) override;
   Status set(Slice key, Slice value) override;
   Status erase(Slice key) override;
   Result<size_t> count(Slice prefix) override;

tddb/td/db/RocksDb.cpp

@@ -65,12 +65,16 @@ Result<RocksDb> RocksDb::open(std::string path, RocksDbOptions options) {
     rocksdb::Options db_options;
 
     static auto default_cache = rocksdb::NewLRUCache(1 << 30);
-    if (options.block_cache == nullptr) {
+    if (!options.no_block_cache && options.block_cache == nullptr) {
       options.block_cache = default_cache;
     }
 
     rocksdb::BlockBasedTableOptions table_options;
-    table_options.block_cache = options.block_cache;
+    if (options.no_block_cache) {
+      table_options.no_block_cache = true;
+    } else {
+      table_options.block_cache = options.block_cache;
+    }
     db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options));
 
     db_options.use_direct_reads = options.use_direct_reads;
@@ -212,6 +216,32 @@ Status RocksDb::for_each(std::function<Status(Slice, Slice)> f) {
   return Status::OK();
 }
 
+Status RocksDb::for_each_in_range(Slice begin, Slice end, std::function<Status(Slice, Slice)> f) {
+  rocksdb::ReadOptions options;
+  options.snapshot = snapshot_.get();
+  std::unique_ptr<rocksdb::Iterator> iterator;
+  if (snapshot_ || !transaction_) {
+    iterator.reset(db_->NewIterator(options));
+  } else {
+    iterator.reset(transaction_->GetIterator(options));
+  }
+
+  auto comparator = rocksdb::BytewiseComparator();
+  iterator->Seek(to_rocksdb(begin));
+  for (; iterator->Valid(); iterator->Next()) {
+    auto key = from_rocksdb(iterator->key());
+    if (comparator->Compare(to_rocksdb(key), to_rocksdb(end)) >= 0) {
+      break;
+    }
+    auto value = from_rocksdb(iterator->value());
+    TRY_STATUS(f(key, value));
+  }
+  if (!iterator->status().ok()) {
+    return from_rocksdb(iterator->status());
+  }
+  return td::Status::OK();
+}
+
 Status RocksDb::begin_write_batch() {
   CHECK(!transaction_);
   write_batch_ = std::make_unique<rocksdb::WriteBatch>();

tddb/td/db/RocksDb.h

@@ -23,6 +23,7 @@
 #endif
 
 #include "td/db/KeyValue.h"
+#include "td/utils/Span.h"
 #include "td/utils/Status.h"
 #include "td/utils/optional.h"
 
@@ -32,6 +33,8 @@
 #include <mutex>
 #include <set>
 
+#include <functional>
+
 namespace rocksdb {
 class Cache;
 class OptimisticTransactionDB;
@@ -59,6 +62,7 @@ struct RocksDbOptions {
   std::shared_ptr<rocksdb::Cache> block_cache;  // Default - one 1G cache for all RocksDb
   std::shared_ptr<RocksDbSnapshotStatistics> snapshot_statistics = nullptr;
   bool use_direct_reads = false;
+  bool no_block_cache = false;
 };
 
 class RocksDb : public KeyValue {
@@ -72,6 +76,7 @@ class RocksDb : public KeyValue {
   Status erase(Slice key) override;
   Result<size_t> count(Slice prefix) override;
   Status for_each(std::function<Status(Slice, Slice)> f) override;
+  Status for_each_in_range (Slice begin, Slice end, std::function<Status(Slice, Slice)> f) override;
 
   Status begin_write_batch() override;
   Status commit_write_batch() override;

tddb/td/db/utils/StreamToFileActor.cpp

@@ -81,7 +81,7 @@ Result<bool> StreamToFileActor::do_loop() {
     sync_at_ = {};
   }
 
-  bool need_update = sync_state_.set_synced_size(synced_size_) | sync_state_.set_flushed_size(flushed_size_);
+  bool need_update = int(sync_state_.set_synced_size(synced_size_)) | int(sync_state_.set_flushed_size(flushed_size_));
   if (need_update && callback_) {
     callback_->on_sync_state_changed();
   }