external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Save rocksdb statistics to file every minute (#932)

Browse source 
* Save rocksdb statistics to file every minute

* Add flag to disable collecting rocksdb statistics
This commit is contained in:
Marat 2024-03-20 12:21:40 +01:00 • committed by GitHub
parent bf9848c60f
commit 7a6bfa7e7a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 85 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

20
validator/db/archive-manager.cpp
View file

@ -601,7 +601,7 @@ void ArchiveManager::load_package(PackageId id) {
}
desc.file =
td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get());
td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_);
m.emplace(id, std::move(desc));
update_permanent_slices();
@ -636,7 +636,7 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull
td::mkdir(db_root_ + id.path()).ensure();
std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
new_desc.file =
td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get());
td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_);
const FileDescription &desc = f.emplace(id, std::move(new_desc));
if (!id.temp) {
update_desc(f, desc, shard, seqno, ts, lt);
@ -829,7 +829,10 @@ void ArchiveManager::start_up() {
if (opts_->get_max_open_archive_files() > 0) {
archive_lru_ = td::actor::create_actor<ArchiveLru>("archive_lru", opts_->get_max_open_archive_files());
}
index_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_root_ + "/files/globalindex").move_as_ok());
if (!opts_->get_disable_rocksdb_stats()) {
statistics_ = td::RocksDb::create_statistics();
}
index_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_root_ + "/files/globalindex", statistics_).move_as_ok());
std::string value;
auto v = index_->get(create_serialize_tl_object<ton_api::db_files_index_key>().as_slice(), value);
v.ensure();
@ -903,6 +906,17 @@ void ArchiveManager::start_up() {
break;
}
}
if (!opts_->get_disable_rocksdb_stats()) {
alarm_timestamp() = td::Timestamp::in(60.0);
}
}
void ArchiveManager::alarm() {
alarm_timestamp() = td::Timestamp::in(60.0);
auto stats = td::RocksDb::statistics_to_string(statistics_);
td::atomic_write_file(db_root_ + "/db_stats.txt", stats);
td::RocksDb::reset_statistics(statistics_);
}
void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) {

3
validator/db/archive-manager.hpp
View file

@ -70,6 +70,7 @@ class ArchiveManager : public td::actor::Actor {
td::Promise<td::BufferSlice> promise);
void start_up() override;
void alarm() override;
void commit_transaction();
void set_async_mode(bool mode, td::Promise<td::Unit> promise);
@ -173,6 +174,8 @@ class ArchiveManager : public td::actor::Actor {
bool huge_transaction_started_ = false;
td::uint32 huge_transaction_size_ = 0;
std::shared_ptr<rocksdb::Statistics> statistics_;
FileMap &get_file_map(const PackageId &p) {
return p.key ? key_files_ : p.temp ? temp_files_ : files_;
}

7
validator/db/archive-slice.cpp
View file

@ -465,7 +465,7 @@ void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::
void ArchiveSlice::before_query() {
if (status_ == st_closed) {
LOG(DEBUG) << "Opening archive slice " << db_path_;
kv_ = std::make_unique<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
kv_ = std::make_unique<td::RocksDb>(td::RocksDb::open(db_path_, statistics_).move_as_ok());
std::string value;
auto R2 = kv_->get("status", value);
R2.ensure();
@ -604,14 +604,15 @@ void ArchiveSlice::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
}
ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root,
td::actor::ActorId<ArchiveLru> archive_lru)
td::actor::ActorId<ArchiveLru> archive_lru, std::shared_ptr<rocksdb::Statistics> statistics)
: archive_id_(archive_id)
, key_blocks_only_(key_blocks_only)
, temp_(temp)
, finalized_(finalized)
, p_id_(archive_id_, key_blocks_only_, temp_)
, db_root_(std::move(db_root))
, archive_lru_(std::move(archive_lru)) {
, archive_lru_(std::move(archive_lru))
, statistics_(statistics) {
db_path_ = PSTRING() << db_root_ << p_id_.path() << p_id_.name() << ".index";
}

7
validator/db/archive-slice.hpp
View file

@ -23,6 +23,10 @@
#include "fileref.hpp"
#include <map>
namespace rocksdb {
class Statistics;
}
namespace ton {
namespace validator {
@ -81,7 +85,7 @@ class ArchiveLru;
class ArchiveSlice : public td::actor::Actor {
public:
ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root,
td::actor::ActorId<ArchiveLru> archive_lru);
td::actor::ActorId<ArchiveLru> archive_lru, std::shared_ptr<rocksdb::Statistics> statistics = nullptr);
void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise);
@ -151,6 +155,7 @@ class ArchiveSlice : public td::actor::Actor {
std::string db_root_;
td::actor::ActorId<ArchiveLru> archive_lru_;
std::shared_ptr<rocksdb::Statistics> statistics_;
std::unique_ptr<td::KeyValue> kv_;
struct PackageInfo {

15
validator/db/celldb.cpp
View file

@ -20,6 +20,7 @@
#include "rootdb.hpp"
#include "td/db/RocksDb.h"
#include "td/utils/filesystem.h"
#include "ton/ton-tl.hpp"
#include "ton/ton-io.hpp"
@ -83,7 +84,12 @@ void CellDbIn::start_up() {
};
CellDbBase::start_up();
cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_).move_as_ok());
if (!opts_->get_disable_rocksdb_stats()) {
statistics_ = td::RocksDb::create_statistics();
statistics_flush_at_ = td::Timestamp::in(60.0);
}
cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_, statistics_).move_as_ok());
boc_ = vm::DynamicBagOfCellsDb::create();
boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth());
@ -156,6 +162,13 @@ void CellDbIn::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>>
}
void CellDbIn::alarm() {
if (statistics_flush_at_ && statistics_flush_at_.is_in_past()) {
statistics_flush_at_ = td::Timestamp::in(60.0);
auto stats = td::RocksDb::statistics_to_string(statistics_);
td::atomic_write_file(path_ + "/db_stats.txt", stats);
td::RocksDb::reset_statistics(statistics_);
}
if (migrate_after_ && migrate_after_.is_in_past()) {
migrate_cells();
}

6
validator/db/celldb.hpp
View file

@ -27,6 +27,10 @@
#include "auto/tl/ton_api.h"
#include "validator.h"
namespace rocksdb {
class Statistics;
}
namespace ton {
namespace validator {
@ -103,6 +107,8 @@ class CellDbIn : public CellDbBase {
std::unique_ptr<vm::DynamicBagOfCellsDb> boc_;
std::shared_ptr<vm::KeyValue> cell_db_;
std::shared_ptr<rocksdb::Statistics> statistics_;
td::Timestamp statistics_flush_at_ = td::Timestamp::never();
std::function<void(const vm::CellLoader::LoadResult&)> on_load_callback_;
std::set<td::Bits256> cells_to_migrate_;

7
validator/validator-options.hpp
View file

@ -123,6 +123,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
double get_archive_preload_period() const override {
return archive_preload_period_;
}
bool get_disable_rocksdb_stats() const override {
return disable_rocksdb_stats_;
}
void set_zero_block_id(BlockIdExt block_id) override {
zero_block_id_ = block_id;
@ -185,6 +188,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
void set_archive_preload_period(double value) override {
archive_preload_period_ = value;
}
void set_disable_rocksdb_stats(bool value) override {
disable_rocksdb_stats_ = value;
}
ValidatorManagerOptionsImpl *make_copy() const override {
return new ValidatorManagerOptionsImpl(*this);
@ -230,6 +236,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
td::uint32 celldb_compress_depth_{0};
size_t max_open_archive_files_ = 0;
double archive_preload_period_ = 0.0;
bool disable_rocksdb_stats_;
};
} // namespace validator

2
validator/validator.h
View file

@ -84,6 +84,7 @@ struct ValidatorManagerOptions : public td::CntObject {
virtual td::uint32 get_celldb_compress_depth() const = 0;
virtual size_t get_max_open_archive_files() const = 0;
virtual double get_archive_preload_period() const = 0;
virtual bool get_disable_rocksdb_stats() const = 0;
virtual void set_zero_block_id(BlockIdExt block_id) = 0;
virtual void set_init_block_id(BlockIdExt block_id) = 0;
@ -106,6 +107,7 @@ struct ValidatorManagerOptions : public td::CntObject {
virtual void set_celldb_compress_depth(td::uint32 value) = 0;
virtual void set_max_open_archive_files(size_t value) = 0;
virtual void set_archive_preload_period(double value) = 0;
virtual void set_disable_rocksdb_stats(bool value) = 0;
static td::Ref<ValidatorManagerOptions> create(
BlockIdExt zero_block_id, BlockIdExt init_block_id,