mirror of
https://github.com/ton-blockchain/ton
synced 2025-02-12 11:12:16 +00:00
Command to disable state serializer (#1011)
Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
parent
229d6a8ee9
commit
74801d00b8
19 changed files with 190 additions and 16 deletions
|
@ -17,6 +17,8 @@
|
||||||
Copyright 2017-2020 Telegram Systems LLP
|
Copyright 2017-2020 Telegram Systems LLP
|
||||||
*/
|
*/
|
||||||
#pragma once
|
#pragma once
|
||||||
|
#include "td/utils/CancellationToken.h"
|
||||||
|
|
||||||
#include <set>
|
#include <set>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include "vm/db/DynamicBagOfCellsDb.h"
|
#include "vm/db/DynamicBagOfCellsDb.h"
|
||||||
|
@ -331,7 +333,7 @@ td::Result<std::vector<Ref<Cell>>> std_boc_deserialize_multi(td::Slice data,
|
||||||
int max_roots = BagOfCells::default_max_roots);
|
int max_roots = BagOfCells::default_max_roots);
|
||||||
td::Result<td::BufferSlice> std_boc_serialize_multi(std::vector<Ref<Cell>> root, int mode = 0);
|
td::Result<td::BufferSlice> std_boc_serialize_multi(std::vector<Ref<Cell>> root, int mode = 0);
|
||||||
|
|
||||||
td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash,
|
td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash, td::FileFd& fd,
|
||||||
td::FileFd& fd, int mode = 0);
|
int mode = 0, td::CancellationToken cancellation_token = {});
|
||||||
|
|
||||||
} // namespace vm
|
} // namespace vm
|
||||||
|
|
|
@ -33,7 +33,8 @@ class LargeBocSerializer {
|
||||||
public:
|
public:
|
||||||
using Hash = Cell::Hash;
|
using Hash = Cell::Hash;
|
||||||
|
|
||||||
explicit LargeBocSerializer(std::shared_ptr<CellDbReader> reader) : reader(std::move(reader)) {
|
explicit LargeBocSerializer(std::shared_ptr<CellDbReader> reader, td::CancellationToken cancellation_token = {})
|
||||||
|
: reader(std::move(reader)), cancellation_token(std::move(cancellation_token)) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void add_root(Hash root);
|
void add_root(Hash root);
|
||||||
|
@ -84,6 +85,7 @@ class LargeBocSerializer {
|
||||||
int revisit(int cell_idx, int force = 0);
|
int revisit(int cell_idx, int force = 0);
|
||||||
td::uint64 compute_sizes(int mode, int& r_size, int& o_size);
|
td::uint64 compute_sizes(int mode, int& r_size, int& o_size);
|
||||||
|
|
||||||
|
td::CancellationToken cancellation_token;
|
||||||
td::Timestamp log_speed_at_;
|
td::Timestamp log_speed_at_;
|
||||||
size_t processed_cells_ = 0;
|
size_t processed_cells_ = 0;
|
||||||
static constexpr double LOG_SPEED_PERIOD = 120.0;
|
static constexpr double LOG_SPEED_PERIOD = 120.0;
|
||||||
|
@ -112,6 +114,9 @@ td::Result<int> LargeBocSerializer::import_cell(Hash hash, int depth) {
|
||||||
return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large");
|
return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large");
|
||||||
}
|
}
|
||||||
++processed_cells_;
|
++processed_cells_;
|
||||||
|
if (processed_cells_ % 1000 == 0) {
|
||||||
|
TRY_STATUS(cancellation_token.check());
|
||||||
|
}
|
||||||
if (log_speed_at_.is_in_past()) {
|
if (log_speed_at_.is_in_past()) {
|
||||||
log_speed_at_ += LOG_SPEED_PERIOD;
|
log_speed_at_ += LOG_SPEED_PERIOD;
|
||||||
LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s";
|
LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s";
|
||||||
|
@ -408,6 +413,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) {
|
||||||
store_ref(k);
|
store_ref(k);
|
||||||
}
|
}
|
||||||
++processed_cells_;
|
++processed_cells_;
|
||||||
|
if (processed_cells_ % 1000 == 0) {
|
||||||
|
TRY_STATUS(cancellation_token.check());
|
||||||
|
}
|
||||||
if (log_speed_at_.is_in_past()) {
|
if (log_speed_at_.is_in_past()) {
|
||||||
log_speed_at_ += LOG_SPEED_PERIOD;
|
log_speed_at_ += LOG_SPEED_PERIOD;
|
||||||
LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s";
|
LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s";
|
||||||
|
@ -428,10 +436,10 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) {
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash, td::FileFd& fd,
|
td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash, td::FileFd& fd,
|
||||||
int mode) {
|
int mode, td::CancellationToken cancellation_token) {
|
||||||
td::Timer timer;
|
td::Timer timer;
|
||||||
CHECK(reader != nullptr)
|
CHECK(reader != nullptr)
|
||||||
LargeBocSerializer serializer(reader);
|
LargeBocSerializer serializer(reader, std::move(cancellation_token));
|
||||||
serializer.add_root(root_hash);
|
serializer.add_root(root_hash);
|
||||||
TRY_STATUS(serializer.import_cells());
|
TRY_STATUS(serializer.import_cells());
|
||||||
TRY_STATUS(serializer.serialize(fd, mode));
|
TRY_STATUS(serializer.serialize(fd, mode));
|
||||||
|
|
|
@ -170,7 +170,7 @@ ton::tl_object_ptr<ton::ton_api::engine_validator_config> Config::tl() const {
|
||||||
return ton::create_tl_object<ton::ton_api::engine_validator_config>(
|
return ton::create_tl_object<ton::ton_api::engine_validator_config>(
|
||||||
out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec),
|
out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec),
|
||||||
ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec),
|
ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec),
|
||||||
nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec));
|
nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec));
|
||||||
}
|
}
|
||||||
|
|
||||||
td::Result<bool> Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip,
|
td::Result<bool> Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip,
|
||||||
|
|
|
@ -20,6 +20,7 @@
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
#include "Status.h"
|
||||||
|
|
||||||
namespace td {
|
namespace td {
|
||||||
|
|
||||||
|
@ -38,6 +39,12 @@ class CancellationToken {
|
||||||
}
|
}
|
||||||
return token_->is_cancelled_.load(std::memory_order_acquire);
|
return token_->is_cancelled_.load(std::memory_order_acquire);
|
||||||
}
|
}
|
||||||
|
Status check() const {
|
||||||
|
if (*this) {
|
||||||
|
return Status::Error(653, "cancelled"); // cancelled = 653
|
||||||
|
}
|
||||||
|
return Status::OK();
|
||||||
|
}
|
||||||
CancellationToken() = default;
|
CancellationToken() = default;
|
||||||
explicit CancellationToken(std::shared_ptr<detail::RawCancellationToken> token) : token_(std::move(token)) {
|
explicit CancellationToken(std::shared_ptr<detail::RawCancellationToken> token) : token_(std::move(token)) {
|
||||||
}
|
}
|
||||||
|
|
|
@ -592,11 +592,13 @@ engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config;
|
||||||
engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster;
|
engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster;
|
||||||
engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave;
|
engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave;
|
||||||
engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig;
|
engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig;
|
||||||
engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl)
|
engine.validator.extraConfig state_serializer_enabled:Bool = engine.validator.ExtraConfig;
|
||||||
|
engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl)
|
||||||
dht:(vector engine.dht)
|
dht:(vector engine.dht)
|
||||||
validators:(vector engine.validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave)
|
validators:(vector engine.validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave)
|
||||||
fullnodemasters:(vector engine.validator.fullNodeMaster)
|
fullnodemasters:(vector engine.validator.fullNodeMaster)
|
||||||
fullnodeconfig:engine.validator.fullNodeConfig
|
fullnodeconfig:engine.validator.fullNodeConfig
|
||||||
|
extraconfig:engine.validator.extraConfig
|
||||||
liteservers:(vector engine.liteServer) control:(vector engine.controlInterface)
|
liteservers:(vector engine.liteServer) control:(vector engine.controlInterface)
|
||||||
gc:engine.gc = engine.validator.Config;
|
gc:engine.gc = engine.validator.Config;
|
||||||
|
|
||||||
|
@ -711,6 +713,8 @@ engine.validator.addCustomOverlay overlay:engine.validator.customOverlay = engin
|
||||||
engine.validator.delCustomOverlay name:string = engine.validator.Success;
|
engine.validator.delCustomOverlay name:string = engine.validator.Success;
|
||||||
engine.validator.showCustomOverlays = engine.validator.CustomOverlaysConfig;
|
engine.validator.showCustomOverlays = engine.validator.CustomOverlaysConfig;
|
||||||
|
|
||||||
|
engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Success;
|
||||||
|
|
||||||
---types---
|
---types---
|
||||||
|
|
||||||
storage.pong = storage.Pong;
|
storage.pong = storage.Pong;
|
||||||
|
|
Binary file not shown.
|
@ -1180,3 +1180,26 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) {
|
||||||
}
|
}
|
||||||
return td::Status::OK();
|
return td::Status::OK();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
td::Status SetStateSerializerEnabledQuery::run() {
|
||||||
|
TRY_RESULT(value, tokenizer_.get_token<int>());
|
||||||
|
if (value != 0 && value != 1) {
|
||||||
|
return td::Status::Error("expected 0 or 1");
|
||||||
|
}
|
||||||
|
TRY_STATUS(tokenizer_.check_endl());
|
||||||
|
enabled_ = value;
|
||||||
|
return td::Status::OK();
|
||||||
|
}
|
||||||
|
|
||||||
|
td::Status SetStateSerializerEnabledQuery::send() {
|
||||||
|
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_setStateSerializerEnabled>(enabled_);
|
||||||
|
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
|
||||||
|
return td::Status::OK();
|
||||||
|
}
|
||||||
|
|
||||||
|
td::Status SetStateSerializerEnabledQuery::receive(td::BufferSlice data) {
|
||||||
|
TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
|
||||||
|
"received incorrect answer: ");
|
||||||
|
td::TerminalIO::out() << "success\n";
|
||||||
|
return td::Status::OK();
|
||||||
|
}
|
||||||
|
|
|
@ -1207,3 +1207,25 @@ class ShowCustomOverlaysQuery : public Query {
|
||||||
return get_name();
|
return get_name();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
class SetStateSerializerEnabledQuery : public Query {
|
||||||
|
public:
|
||||||
|
SetStateSerializerEnabledQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
|
||||||
|
: Query(console, std::move(tokenizer)) {
|
||||||
|
}
|
||||||
|
td::Status run() override;
|
||||||
|
td::Status send() override;
|
||||||
|
td::Status receive(td::BufferSlice data) override;
|
||||||
|
static std::string get_name() {
|
||||||
|
return "setstateserializerenabled";
|
||||||
|
}
|
||||||
|
static std::string get_help() {
|
||||||
|
return "setstateserializerenabled <value>\tdisable or enable persistent state serializer; value is 0 or 1";
|
||||||
|
}
|
||||||
|
std::string name() const override {
|
||||||
|
return get_name();
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool enabled_;
|
||||||
|
};
|
||||||
|
|
|
@ -146,6 +146,7 @@ void ValidatorEngineConsole::run() {
|
||||||
add_query_runner(std::make_unique<QueryRunnerImpl<AddCustomOverlayQuery>>());
|
add_query_runner(std::make_unique<QueryRunnerImpl<AddCustomOverlayQuery>>());
|
||||||
add_query_runner(std::make_unique<QueryRunnerImpl<DelCustomOverlayQuery>>());
|
add_query_runner(std::make_unique<QueryRunnerImpl<DelCustomOverlayQuery>>());
|
||||||
add_query_runner(std::make_unique<QueryRunnerImpl<ShowCustomOverlaysQuery>>());
|
add_query_runner(std::make_unique<QueryRunnerImpl<ShowCustomOverlaysQuery>>());
|
||||||
|
add_query_runner(std::make_unique<QueryRunnerImpl<SetStateSerializerEnabledQuery>>());
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise<td::BufferSlice> promise) {
|
bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise<td::BufferSlice> promise) {
|
||||||
|
|
|
@ -73,6 +73,7 @@
|
||||||
#include "block-parse.h"
|
#include "block-parse.h"
|
||||||
#include "common/delay.h"
|
#include "common/delay.h"
|
||||||
#include "block/precompiled-smc/PrecompiledSmartContract.h"
|
#include "block/precompiled-smc/PrecompiledSmartContract.h"
|
||||||
|
#include "interfaces/validator-manager.h"
|
||||||
|
|
||||||
Config::Config() {
|
Config::Config() {
|
||||||
out_port = 3278;
|
out_port = 3278;
|
||||||
|
@ -155,6 +156,11 @@ Config::Config(ton::ton_api::engine_validator_config &config) {
|
||||||
if (config.fullnodeconfig_) {
|
if (config.fullnodeconfig_) {
|
||||||
full_node_config = ton::validator::fullnode::FullNodeConfig(config.fullnodeconfig_);
|
full_node_config = ton::validator::fullnode::FullNodeConfig(config.fullnodeconfig_);
|
||||||
}
|
}
|
||||||
|
if (config.extraconfig_) {
|
||||||
|
state_serializer_enabled = config.extraconfig_->state_serializer_enabled_;
|
||||||
|
} else {
|
||||||
|
state_serializer_enabled = true;
|
||||||
|
}
|
||||||
|
|
||||||
for (auto &serv : config.liteservers_) {
|
for (auto &serv : config.liteservers_) {
|
||||||
config_add_lite_server(ton::PublicKeyHash{serv->id_}, serv->port_).ensure();
|
config_add_lite_server(ton::PublicKeyHash{serv->id_}, serv->port_).ensure();
|
||||||
|
@ -231,6 +237,12 @@ ton::tl_object_ptr<ton::ton_api::engine_validator_config> Config::tl() const {
|
||||||
full_node_config_obj = full_node_config.tl();
|
full_node_config_obj = full_node_config.tl();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ton::tl_object_ptr<ton::ton_api::engine_validator_extraConfig> extra_config_obj = {};
|
||||||
|
if (!state_serializer_enabled) {
|
||||||
|
// Non-default values
|
||||||
|
extra_config_obj = ton::create_tl_object<ton::ton_api::engine_validator_extraConfig>(state_serializer_enabled);
|
||||||
|
}
|
||||||
|
|
||||||
std::vector<ton::tl_object_ptr<ton::ton_api::engine_liteServer>> liteserver_vec;
|
std::vector<ton::tl_object_ptr<ton::ton_api::engine_liteServer>> liteserver_vec;
|
||||||
for (auto &x : liteservers) {
|
for (auto &x : liteservers) {
|
||||||
liteserver_vec.push_back(ton::create_tl_object<ton::ton_api::engine_liteServer>(x.second.tl(), x.first));
|
liteserver_vec.push_back(ton::create_tl_object<ton::ton_api::engine_liteServer>(x.second.tl(), x.first));
|
||||||
|
@ -253,7 +265,7 @@ ton::tl_object_ptr<ton::ton_api::engine_validator_config> Config::tl() const {
|
||||||
return ton::create_tl_object<ton::ton_api::engine_validator_config>(
|
return ton::create_tl_object<ton::ton_api::engine_validator_config>(
|
||||||
out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), full_node.tl(),
|
out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), full_node.tl(),
|
||||||
std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(full_node_config_obj),
|
std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(full_node_config_obj),
|
||||||
std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec));
|
std::move(extra_config_obj), std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec));
|
||||||
}
|
}
|
||||||
|
|
||||||
td::Result<bool> Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip,
|
td::Result<bool> Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip,
|
||||||
|
@ -1399,6 +1411,7 @@ td::Status ValidatorEngine::load_global_config() {
|
||||||
h.push_back(b);
|
h.push_back(b);
|
||||||
}
|
}
|
||||||
validator_options_.write().set_hardforks(std::move(h));
|
validator_options_.write().set_hardforks(std::move(h));
|
||||||
|
validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled);
|
||||||
|
|
||||||
return td::Status::OK();
|
return td::Status::OK();
|
||||||
}
|
}
|
||||||
|
@ -3642,6 +3655,34 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCusto
|
||||||
custom_overlays_config_, true));
|
custom_overlays_config_, true));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query,
|
||||||
|
td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
|
||||||
|
td::Promise<td::BufferSlice> promise) {
|
||||||
|
if (!(perm & ValidatorEnginePermissions::vep_modify)) {
|
||||||
|
promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!started_) {
|
||||||
|
promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started")));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (query.enabled_ == validator_options_->get_state_serializer_enabled()) {
|
||||||
|
promise.set_value(ton::create_serialize_tl_object<ton::ton_api::engine_validator_success>());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
validator_options_.write().set_state_serializer_enabled(query.enabled_);
|
||||||
|
td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options,
|
||||||
|
validator_options_);
|
||||||
|
config_.state_serializer_enabled = query.enabled_;
|
||||||
|
write_config([promise = std::move(promise)](td::Result<td::Unit> R) mutable {
|
||||||
|
if (R.is_error()) {
|
||||||
|
promise.set_value(create_control_query_error(R.move_as_error()));
|
||||||
|
} else {
|
||||||
|
promise.set_value(ton::create_serialize_tl_object<ton::ton_api::engine_validator_success>());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src,
|
void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src,
|
||||||
ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data,
|
ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data,
|
||||||
td::Promise<td::BufferSlice> promise) {
|
td::Promise<td::BufferSlice> promise) {
|
||||||
|
|
|
@ -90,6 +90,8 @@ struct Config {
|
||||||
std::map<td::int32, Control> controls;
|
std::map<td::int32, Control> controls;
|
||||||
std::set<ton::PublicKeyHash> gc;
|
std::set<ton::PublicKeyHash> gc;
|
||||||
|
|
||||||
|
bool state_serializer_enabled = true;
|
||||||
|
|
||||||
void decref(ton::PublicKeyHash key);
|
void decref(ton::PublicKeyHash key);
|
||||||
void incref(ton::PublicKeyHash key) {
|
void incref(ton::PublicKeyHash key) {
|
||||||
keys_refcnt[key]++;
|
keys_refcnt[key]++;
|
||||||
|
@ -473,6 +475,8 @@ class ValidatorEngine : public td::actor::Actor {
|
||||||
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
|
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
|
||||||
void run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data,
|
void run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data,
|
||||||
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
|
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
|
||||||
|
void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data,
|
||||||
|
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
|
||||||
template <class T>
|
template <class T>
|
||||||
void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
|
void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
|
||||||
td::Promise<td::BufferSlice> promise) {
|
td::Promise<td::BufferSlice> promise) {
|
||||||
|
|
|
@ -422,6 +422,10 @@ class ValidatorManagerImpl : public ValidatorManager {
|
||||||
promise.set_result(td::Status::Error("not implemented"));
|
promise.set_result(td::Status::Error("not implemented"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void update_options(td::Ref<ValidatorManagerOptions> opts) override {
|
||||||
|
opts_ = std::move(opts);
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
PublicKeyHash local_id_;
|
PublicKeyHash local_id_;
|
||||||
|
|
||||||
|
|
|
@ -483,6 +483,9 @@ class ValidatorManagerImpl : public ValidatorManager {
|
||||||
td::Promise<tl_object_ptr<lite_api::liteServer_nonfinal_validatorGroups>> promise) override {
|
td::Promise<tl_object_ptr<lite_api::liteServer_nonfinal_validatorGroups>> promise) override {
|
||||||
promise.set_result(td::Status::Error("not implemented"));
|
promise.set_result(td::Status::Error("not implemented"));
|
||||||
}
|
}
|
||||||
|
void update_options(td::Ref<ValidatorManagerOptions> opts) override {
|
||||||
|
opts_ = std::move(opts);
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
td::Ref<ValidatorManagerOptions> opts_;
|
td::Ref<ValidatorManagerOptions> opts_;
|
||||||
|
|
|
@ -3132,6 +3132,14 @@ void ValidatorManagerImpl::get_validator_groups_info_for_litequery(
|
||||||
td::actor::create_actor<Actor>("get-validator-groups-info", std::move(groups), std::move(promise)).release();
|
td::actor::create_actor<Actor>("get-validator-groups-info", std::move(groups), std::move(promise)).release();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ValidatorManagerImpl::update_options(td::Ref<ValidatorManagerOptions> opts) {
|
||||||
|
// Currently options can be updated only to change state_serializer_enabled flag
|
||||||
|
if (!serializer_.empty()) {
|
||||||
|
td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts);
|
||||||
|
}
|
||||||
|
opts_ = std::move(opts);
|
||||||
|
}
|
||||||
|
|
||||||
td::actor::ActorOwn<ValidatorManagerInterface> ValidatorManagerFactory::create(
|
td::actor::ActorOwn<ValidatorManagerInterface> ValidatorManagerFactory::create(
|
||||||
td::Ref<ValidatorManagerOptions> opts, std::string db_root, td::actor::ActorId<keyring::Keyring> keyring,
|
td::Ref<ValidatorManagerOptions> opts, std::string db_root, td::actor::ActorId<keyring::Keyring> keyring,
|
||||||
td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<rldp::Rldp> rldp,
|
td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<rldp::Rldp> rldp,
|
||||||
|
|
|
@ -588,6 +588,8 @@ class ValidatorManagerImpl : public ValidatorManager {
|
||||||
void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override;
|
void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override;
|
||||||
void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override;
|
void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override;
|
||||||
|
|
||||||
|
void update_options(td::Ref<ValidatorManagerOptions> opts) override;
|
||||||
|
|
||||||
void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint32> promise) override {
|
void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint32> promise) override {
|
||||||
if (queue_size_counter_.empty()) {
|
if (queue_size_counter_.empty()) {
|
||||||
if (last_masterchain_state_.is_null()) {
|
if (last_masterchain_state_.is_null()) {
|
||||||
|
|
|
@ -27,6 +27,9 @@ namespace ton {
|
||||||
namespace validator {
|
namespace validator {
|
||||||
|
|
||||||
void AsyncStateSerializer::start_up() {
|
void AsyncStateSerializer::start_up() {
|
||||||
|
if (!opts_->get_state_serializer_enabled()) {
|
||||||
|
LOG(ERROR) << "Persistent state serializer is disabled";
|
||||||
|
}
|
||||||
alarm_timestamp() = td::Timestamp::in(1.0 + td::Random::fast(0, 10) * 1.0);
|
alarm_timestamp() = td::Timestamp::in(1.0 + td::Random::fast(0, 10) * 1.0);
|
||||||
running_ = true;
|
running_ = true;
|
||||||
|
|
||||||
|
@ -130,7 +133,7 @@ void AsyncStateSerializer::next_iteration() {
|
||||||
}
|
}
|
||||||
CHECK(masterchain_handle_->id() == last_block_id_);
|
CHECK(masterchain_handle_->id() == last_block_id_);
|
||||||
if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno &&
|
if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno &&
|
||||||
need_serialize(masterchain_handle_)) {
|
need_serialize(masterchain_handle_) && opts_->get_state_serializer_enabled()) {
|
||||||
if (!have_masterchain_state_) {
|
if (!have_masterchain_state_) {
|
||||||
LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str();
|
LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str();
|
||||||
// block next attempts immediately, but send actual request later
|
// block next attempts immediately, but send actual request later
|
||||||
|
@ -174,6 +177,9 @@ void AsyncStateSerializer::next_iteration() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (masterchain_handle_->inited_next_left()) {
|
if (masterchain_handle_->inited_next_left()) {
|
||||||
|
if (need_serialize(masterchain_handle_) && !opts_->get_state_serializer_enabled()) {
|
||||||
|
LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str();
|
||||||
|
}
|
||||||
last_block_id_ = masterchain_handle_->one_next(true);
|
last_block_id_ = masterchain_handle_->one_next(true);
|
||||||
have_masterchain_state_ = false;
|
have_masterchain_state_ = false;
|
||||||
masterchain_handle_ = nullptr;
|
masterchain_handle_ = nullptr;
|
||||||
|
@ -200,6 +206,10 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) {
|
||||||
|
|
||||||
void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state,
|
void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state,
|
||||||
std::shared_ptr<vm::CellDbReader> cell_db_reader) {
|
std::shared_ptr<vm::CellDbReader> cell_db_reader) {
|
||||||
|
if (!opts_->get_state_serializer_enabled()) {
|
||||||
|
stored_masterchain_state();
|
||||||
|
return;
|
||||||
|
}
|
||||||
LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str();
|
LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str();
|
||||||
have_masterchain_state_ = true;
|
have_masterchain_state_ = true;
|
||||||
CHECK(next_idx_ == 0);
|
CHECK(next_idx_ == 0);
|
||||||
|
@ -210,11 +220,16 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state
|
||||||
shards_.push_back(v->top_block_id());
|
shards_.push_back(v->top_block_id());
|
||||||
}
|
}
|
||||||
|
|
||||||
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) {
|
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader,
|
||||||
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31);
|
cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable {
|
||||||
|
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token));
|
||||||
};
|
};
|
||||||
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
|
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
|
||||||
R.ensure();
|
if (R.is_error() && R.error().code() == cancelled) {
|
||||||
|
LOG(ERROR) << "Persistent state serialization cancelled";
|
||||||
|
} else {
|
||||||
|
R.ensure();
|
||||||
|
}
|
||||||
td::actor::send_closure(SelfId, &AsyncStateSerializer::stored_masterchain_state);
|
td::actor::send_closure(SelfId, &AsyncStateSerializer::stored_masterchain_state);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -253,13 +268,22 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) {
|
||||||
|
|
||||||
void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardState> state,
|
void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardState> state,
|
||||||
std::shared_ptr<vm::CellDbReader> cell_db_reader) {
|
std::shared_ptr<vm::CellDbReader> cell_db_reader) {
|
||||||
|
if (!opts_->get_state_serializer_enabled()) {
|
||||||
|
success_handler();
|
||||||
|
return;
|
||||||
|
}
|
||||||
LOG(ERROR) << "serializing shard state " << handle->id().id.to_str();
|
LOG(ERROR) << "serializing shard state " << handle->id().id.to_str();
|
||||||
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) {
|
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader,
|
||||||
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31);
|
cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable {
|
||||||
|
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token));
|
||||||
};
|
};
|
||||||
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Unit> R) {
|
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Unit> R) {
|
||||||
R.ensure();
|
if (R.is_error() && R.error().code() == cancelled) {
|
||||||
LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str();
|
LOG(ERROR) << "Persistent state serialization cancelled";
|
||||||
|
} else {
|
||||||
|
R.ensure();
|
||||||
|
LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str();
|
||||||
|
}
|
||||||
td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler);
|
td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler);
|
||||||
});
|
});
|
||||||
td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(),
|
td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(),
|
||||||
|
@ -285,6 +309,14 @@ void AsyncStateSerializer::success_handler() {
|
||||||
next_iteration();
|
next_iteration();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void AsyncStateSerializer::update_options(td::Ref<ValidatorManagerOptions> opts) {
|
||||||
|
opts_ = std::move(opts);
|
||||||
|
if (!opts_->get_state_serializer_enabled()) {
|
||||||
|
cancellation_token_source_.cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
bool AsyncStateSerializer::need_monitor(ShardIdFull shard) {
|
bool AsyncStateSerializer::need_monitor(ShardIdFull shard) {
|
||||||
return opts_->need_monitor(shard);
|
return opts_->need_monitor(shard);
|
||||||
}
|
}
|
||||||
|
|
|
@ -37,6 +37,7 @@ class AsyncStateSerializer : public td::actor::Actor {
|
||||||
bool saved_to_db_ = true;
|
bool saved_to_db_ = true;
|
||||||
|
|
||||||
td::Ref<ValidatorManagerOptions> opts_;
|
td::Ref<ValidatorManagerOptions> opts_;
|
||||||
|
td::CancellationTokenSource cancellation_token_source_;
|
||||||
|
|
||||||
td::actor::ActorId<ValidatorManager> manager_;
|
td::actor::ActorId<ValidatorManager> manager_;
|
||||||
|
|
||||||
|
@ -89,6 +90,8 @@ class AsyncStateSerializer : public td::actor::Actor {
|
||||||
void fail_handler(td::Status reason);
|
void fail_handler(td::Status reason);
|
||||||
void fail_handler_cont();
|
void fail_handler_cont();
|
||||||
void success_handler();
|
void success_handler();
|
||||||
|
|
||||||
|
void update_options(td::Ref<ValidatorManagerOptions> opts);
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace validator
|
} // namespace validator
|
||||||
|
|
|
@ -141,6 +141,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
|
||||||
td::optional<double> get_catchain_max_block_delay() const override {
|
td::optional<double> get_catchain_max_block_delay() const override {
|
||||||
return catchain_max_block_delay_;
|
return catchain_max_block_delay_;
|
||||||
}
|
}
|
||||||
|
bool get_state_serializer_enabled() const override {
|
||||||
|
return state_serializer_enabled_;
|
||||||
|
}
|
||||||
|
|
||||||
void set_zero_block_id(BlockIdExt block_id) override {
|
void set_zero_block_id(BlockIdExt block_id) override {
|
||||||
zero_block_id_ = block_id;
|
zero_block_id_ = block_id;
|
||||||
|
@ -221,6 +224,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
|
||||||
void set_catchain_max_block_delay(double value) override {
|
void set_catchain_max_block_delay(double value) override {
|
||||||
catchain_max_block_delay_ = value;
|
catchain_max_block_delay_ = value;
|
||||||
}
|
}
|
||||||
|
void set_state_serializer_enabled(bool value) override {
|
||||||
|
state_serializer_enabled_ = value;
|
||||||
|
}
|
||||||
|
|
||||||
ValidatorManagerOptionsImpl *make_copy() const override {
|
ValidatorManagerOptionsImpl *make_copy() const override {
|
||||||
return new ValidatorManagerOptionsImpl(*this);
|
return new ValidatorManagerOptionsImpl(*this);
|
||||||
|
@ -272,6 +278,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
|
||||||
bool celldb_direct_io_ = false;
|
bool celldb_direct_io_ = false;
|
||||||
bool celldb_preload_all_ = false;
|
bool celldb_preload_all_ = false;
|
||||||
td::optional<double> catchain_max_block_delay_;
|
td::optional<double> catchain_max_block_delay_;
|
||||||
|
bool state_serializer_enabled_ = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace validator
|
} // namespace validator
|
||||||
|
|
|
@ -90,6 +90,7 @@ struct ValidatorManagerOptions : public td::CntObject {
|
||||||
virtual bool get_celldb_direct_io() const = 0;
|
virtual bool get_celldb_direct_io() const = 0;
|
||||||
virtual bool get_celldb_preload_all() const = 0;
|
virtual bool get_celldb_preload_all() const = 0;
|
||||||
virtual td::optional<double> get_catchain_max_block_delay() const = 0;
|
virtual td::optional<double> get_catchain_max_block_delay() const = 0;
|
||||||
|
virtual bool get_state_serializer_enabled() const = 0;
|
||||||
|
|
||||||
virtual void set_zero_block_id(BlockIdExt block_id) = 0;
|
virtual void set_zero_block_id(BlockIdExt block_id) = 0;
|
||||||
virtual void set_init_block_id(BlockIdExt block_id) = 0;
|
virtual void set_init_block_id(BlockIdExt block_id) = 0;
|
||||||
|
@ -118,6 +119,7 @@ struct ValidatorManagerOptions : public td::CntObject {
|
||||||
virtual void set_celldb_direct_io(bool value) = 0;
|
virtual void set_celldb_direct_io(bool value) = 0;
|
||||||
virtual void set_celldb_preload_all(bool value) = 0;
|
virtual void set_celldb_preload_all(bool value) = 0;
|
||||||
virtual void set_catchain_max_block_delay(double value) = 0;
|
virtual void set_catchain_max_block_delay(double value) = 0;
|
||||||
|
virtual void set_state_serializer_enabled(bool value) = 0;
|
||||||
|
|
||||||
static td::Ref<ValidatorManagerOptions> create(
|
static td::Ref<ValidatorManagerOptions> create(
|
||||||
BlockIdExt zero_block_id, BlockIdExt init_block_id,
|
BlockIdExt zero_block_id, BlockIdExt init_block_id,
|
||||||
|
@ -249,6 +251,7 @@ class ValidatorManagerInterface : public td::actor::Actor {
|
||||||
virtual void add_perf_timer_stat(std::string name, double duration) = 0;
|
virtual void add_perf_timer_stat(std::string name, double duration) = 0;
|
||||||
virtual void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint32> promise) = 0;
|
virtual void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint32> promise) = 0;
|
||||||
|
|
||||||
|
virtual void update_options(td::Ref<ValidatorManagerOptions> opts) = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace validator
|
} // namespace validator
|
||||||
|
|
Loading…
Reference in a new issue