external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Limit file descriptors num by adding archive slice lru (#892)

Browse source 
* --max-archive-fd option limits open files in archive manager

* Don't close the latest archives + bugfix

* Delete temp packages early

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK 2024-02-07 14:56:37 +03:00 • committed by GitHub
parent e723213d5c
commit 12c1b1a2e6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 316 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

39
validator/db/archive-manager.cpp
View file

@ -55,7 +55,9 @@ std::string PackageId::name() const {
}
}
ArchiveManager::ArchiveManager(td::actor::ActorId<RootDb> root, std::string db_root) : db_root_(db_root) {
ArchiveManager::ArchiveManager(td::actor::ActorId<RootDb> root, std::string db_root,
td::Ref<ValidatorManagerOptions> opts)
: db_root_(db_root), opts_(opts) {
}
void ArchiveManager::add_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
@ -598,9 +600,11 @@ void ArchiveManager::load_package(PackageId id) {
}
}
desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_);
desc.file =
td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get());
m.emplace(id, std::move(desc));
update_permanent_slices();
}
const ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno,
@ -631,7 +635,8 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull
FileDescription new_desc{id, false};
td::mkdir(db_root_ + id.path()).ensure();
std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
new_desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_);
new_desc.file =
td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get());
const FileDescription &desc = f.emplace(id, std::move(new_desc));
if (!id.temp) {
update_desc(f, desc, shard, seqno, ts, lt);
@ -673,6 +678,7 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull
.ensure();
}
index_->commit_transaction().ensure();
update_permanent_slices();
return &desc;
}
@ -820,6 +826,9 @@ void ArchiveManager::start_up() {
td::mkdir(db_root_ + "/archive/states/").ensure();
td::mkdir(db_root_ + "/files/").ensure();
td::mkdir(db_root_ + "/files/packages/").ensure();
if (opts_->get_max_open_archive_files() > 0) {
archive_lru_ = td::actor::create_actor<ArchiveLru>("archive_lru", opts_->get_max_open_archive_files());
}
index_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_root_ + "/files/globalindex").move_as_ok());
std::string value;
auto v = index_->get(create_serialize_tl_object<ton_api::db_files_index_key>().as_slice(), value);
@ -878,8 +887,8 @@ void ArchiveManager::start_up() {
persistent_state_gc(FileHash::zero());
}
void ArchiveManager::run_gc(UnixTime ts, UnixTime archive_ttl) {
auto p = get_temp_package_id_by_unixtime(ts);
void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) {
auto p = get_temp_package_id_by_unixtime(std::max(gc_ts, mc_ts - TEMP_PACKAGES_TTL));
std::vector<PackageId> vec;
for (auto &x : temp_files_) {
if (x.first < p) {
@ -907,7 +916,7 @@ void ArchiveManager::run_gc(UnixTime ts, UnixTime archive_ttl) {
if (it == desc.first_blocks.end()) {
continue;
}
if (it->second.ts < ts - archive_ttl) {
if (it->second.ts < gc_ts - archive_ttl) {
vec.push_back(f.first);
}
}
@ -1200,6 +1209,7 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han
}
}
}
update_permanent_slices();
}
void ArchiveManager::FileMap::shard_index_add(const FileDescription &desc) {
@ -1298,6 +1308,23 @@ const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_next_file_de
return it2->second->deleted ? nullptr : it2->second;
}
void ArchiveManager::update_permanent_slices() {
if (archive_lru_.empty()) {
return;
}
std::vector<PackageId> ids;
if (!files_.empty()) {
ids.push_back(files_.rbegin()->first);
}
if (!key_files_.empty()) {
ids.push_back(key_files_.rbegin()->first);
}
if (!temp_files_.empty()) {
ids.push_back(temp_files_.rbegin()->first);
}
td::actor::send_closure(archive_lru_, &ArchiveLru::set_permanent_slices, std::move(ids));
}
} // namespace validator
} // namespace ton