external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Optimization of persistent state serialization (#364)

Browse source 
* Fix double serialization of masterchain; increase sync_blocks_before

* Improve logging in DownloadState

* Write persistent state directly to file instead of a buffer

* Don't keep ref to masterchain state in AsyncStateSerializer

* Sparse state serialization over longer period

Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK 2022-05-15 17:51:24 +03:00 • committed by GitHub
parent 56f0293650
commit c07394aab5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 327 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

25
validator/db/archive-manager.cpp
View file

@ -336,6 +336,28 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t
void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data,
td::Promise<td::Unit> promise) {
auto create_writer = [&](std::string path, td::Promise<std::string> P) {
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/",
std::move(path), std::move(data), std::move(P))
.release();
};
add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer));
}
void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
std::function<td::Status(td::FileFd&)> write_state,
td::Promise<td::Unit> promise) {
auto create_writer = [&](std::string path, td::Promise<std::string> P) {
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/",
std::move(path), std::move(write_state), std::move(P))
.release();
};
add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer));
}
void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id,
td::Promise<td::Unit> promise,
std::function<void(std::string, td::Promise<std::string>)> create_writer) {
auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
auto hash = id.hash();
if (perm_states_.find(hash) != perm_states_.end()) {
@ -353,8 +375,7 @@ void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt master
promise.set_value(td::Unit());
}
});
td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/", path, std::move(data), std::move(P))
.release();
create_writer(std::move(path), std::move(P));
}
void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {

5
validator/db/archive-manager.hpp
View file

@ -45,6 +45,9 @@ class ArchiveManager : public td::actor::Actor {
void add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise<td::Unit> promise);
void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data,
td::Promise<td::Unit> promise);
void add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
std::function<td::Status(td::FileFd&)> write_state,
td::Promise<td::Unit> promise);
void get_zero_state(BlockIdExt block_id, td::Promise<td::BufferSlice> promise);
void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::BufferSlice> promise);
void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,
@ -137,6 +140,8 @@ class ArchiveManager : public td::actor::Actor {
PackageId get_max_temp_file_desc_idx();
PackageId get_prev_temp_file_desc_idx(PackageId id);
void add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise,
std::function<void(std::string, td::Promise<std::string>)> create_writer);
void written_perm_state(FileReferenceShort id);
void persistent_state_gc(FileHash last);

42
validator/db/files-async.hpp
View file

@ -52,30 +52,50 @@ class WriteFile : public td::actor::Actor {
auto res = R.move_as_ok();
auto file = std::move(res.first);
auto old_name = res.second;
td::uint64 offset = 0;
while (data_.size() > 0) {
auto R = file.pwrite(data_.as_slice(), offset);
auto s = R.move_as_ok();
offset += s;
data_.confirm_read(s);
auto status = write_data_(file);
if (!status.is_error()) {
status = file.sync();
}
if (status.is_error()) {
td::unlink(old_name);
promise_.set_error(std::move(status));
stop();
return;
}
file.sync().ensure();
if (new_name_.length() > 0) {
td::rename(old_name, new_name_).ensure();
promise_.set_value(std::move(new_name_));
status = td::rename(old_name, new_name_);
if (status.is_error()) {
promise_.set_error(std::move(status));
} else {
promise_.set_value(std::move(new_name_));
}
} else {
promise_.set_value(std::move(old_name));
}
stop();
}
WriteFile(std::string tmp_dir, std::string new_name, std::function<td::Status(td::FileFd&)> write_data,
td::Promise<std::string> promise)
: tmp_dir_(tmp_dir), new_name_(new_name), write_data_(std::move(write_data)), promise_(std::move(promise)) {
}
WriteFile(std::string tmp_dir, std::string new_name, td::BufferSlice data, td::Promise<std::string> promise)
: tmp_dir_(tmp_dir), new_name_(new_name), data_(std::move(data)), promise_(std::move(promise)) {
: tmp_dir_(tmp_dir), new_name_(new_name), promise_(std::move(promise)) {
write_data_ = [data_ptr = std::make_shared<td::BufferSlice>(std::move(data))] (td::FileFd& fd) {
auto data = std::move(*data_ptr);
td::uint64 offset = 0;
while (data.size() > 0) {
TRY_RESULT(s, fd.pwrite(data.as_slice(), offset));
offset += s;
data.confirm_read(s);
}
return td::Status::OK();
};
}
private:
const std::string tmp_dir_;
std::string new_name_;
td::BufferSlice data_;
std::function<td::Status(td::FileFd&)> write_data_;
td::Promise<std::string> promise_;
};

7
validator/db/rootdb.cpp
View file

@ -276,6 +276,13 @@ void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterc
std::move(state), std::move(promise));
}
void RootDb::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
std::function<td::Status(td::FileFd&)> write_data,
td::Promise<td::Unit> promise) {
td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state_gen, block_id, masterchain_block_id,
std::move(write_data), std::move(promise));
}
void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id,
td::Promise<td::BufferSlice> promise) {
td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id,

3
validator/db/rootdb.hpp
View file

@ -69,6 +69,9 @@ class RootDb : public Db {
void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
td::Promise<td::Unit> promise) override;
void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
std::function<td::Status(td::FileFd&)> write_data,
td::Promise<td::Unit> promise) override;
void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id,
td::Promise<td::BufferSlice> promise) override;
void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,