external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Persistent optimization (#382)

Browse source 
* State serialization consuming less RAM

* Move large-boc-serializer to crypto/vm, refactor code

Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK 2022-05-24 21:17:11 +03:00 • committed by GitHub
parent c04eb8e4f3
commit 89b8717cf8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 665 additions and 157 deletions
Download patch file Download diff file Expand all files Collapse all files

71
validator/state-serializer.cpp
View file

@ -82,24 +82,24 @@ void AsyncStateSerializer::alarm() {
}
void AsyncStateSerializer::request_masterchain_state() {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler,
R.move_as_error_prefix("failed to get masterchain state: "));
} else {
td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state,
td::Ref<MasterchainState>(R.move_as_ok()));
}
});
td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P));
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler,
R.move_as_error_prefix("failed to get masterchain state: "));
} else {
td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state,
td::Ref<MasterchainState>(R.move_as_ok()));
}
});
td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P));
}
void AsyncStateSerializer::request_shard_state(BlockIdExt shard) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
R.ensure();
td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok());
});
return td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shard, true, std::move(P));
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
R.ensure();
td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok());
});
return td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shard, true, std::move(P));
}
void AsyncStateSerializer::next_iteration() {
@ -122,8 +122,21 @@ void AsyncStateSerializer::next_iteration() {
CHECK(masterchain_handle_->id() == last_block_id_);
if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno &&
need_serialize(masterchain_handle_)) {
if (!cell_db_reader_) {
running_ = true;
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::shared_ptr<vm::CellDbReader>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler,
R.move_as_error_prefix("failed to get cell db reader: "));
} else {
td::actor::send_closure(SelfId, &AsyncStateSerializer::got_cell_db_reader, R.move_as_ok());
}
});
td::actor::send_closure(manager_, &ValidatorManager::get_cell_db_reader, std::move(P));
return;
}
if (!have_masterchain_state_) {
LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().seqno();
LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().id;
// block next attempts immediately, but send actual request later
running_ = true;
delay_action(
@ -143,9 +156,10 @@ void AsyncStateSerializer::next_iteration() {
return;
}
}
LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().seqno();
LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().id;
last_key_block_ts_ = masterchain_handle_->unix_time();
last_key_block_id_ = masterchain_handle_->id();
cell_db_reader_ = nullptr;
}
if (!saved_to_db_) {
running_ = true;
@ -175,6 +189,13 @@ void AsyncStateSerializer::got_top_masterchain_handle(BlockIdExt block_id) {
}
}
void AsyncStateSerializer::got_cell_db_reader(std::shared_ptr<vm::CellDbReader> cell_db_reader) {
cell_db_reader_ = std::move(cell_db_reader);
running_ = false;
attempt_ = 0;
next_iteration();
}
void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) {
CHECK(!masterchain_handle_);
masterchain_handle_ = std::move(handle);
@ -184,7 +205,7 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) {
}
void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state) {
LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().seqno();
LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().id;
have_masterchain_state_ = true;
CHECK(next_idx_ == 0);
CHECK(shards_.size() == 0);
@ -194,8 +215,8 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state
shards_.push_back(v->top_block_id());
}
auto write_data = [state] (td::FileFd& fd) {
return state->serialize_to_file(fd);
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = cell_db_reader_] (td::FileFd& fd) {
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31);
};
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
R.ensure();
@ -207,7 +228,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state
}
void AsyncStateSerializer::stored_masterchain_state() {
LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().seqno();
LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().id;
running_ = false;
next_iteration();
}
@ -225,13 +246,13 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) {
}
void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardState> state) {
LOG(INFO) << "serializing shard state " << handle->id().seqno();
auto write_data = [state] (td::FileFd& fd) {
return state->serialize_to_file(fd);
LOG(INFO) << "serializing shard state " << handle->id().id;
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = cell_db_reader_] (td::FileFd& fd) {
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31);
};
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Unit> R) {
R.ensure();
LOG(INFO) << "finished serializing shard state " << handle->id().seqno();
LOG(INFO) << "finished serializing shard state " << handle->id().id;
td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler);
});
td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(),