external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Merge branch 'testnet' into block-generation

Browse source
This commit is contained in:
SpyCheese 2024-06-04 16:49:47 +03:00
commit eb4c876f22
21 changed files with 243 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crypto/vm/boc.h
View file

@ -17,6 +17,8 @@
Copyright 2017-2020 Telegram Systems LLP
*/
#pragma once
#include "td/utils/CancellationToken.h"
#include <set>
#include <map>
#include "vm/db/DynamicBagOfCellsDb.h"
@ -343,7 +345,7 @@ td::Result<std::vector<Ref<Cell>>> std_boc_deserialize_multi(td::Slice data,
int max_roots = BagOfCells::default_max_roots);
td::Result<td::BufferSlice> std_boc_serialize_multi(std::vector<Ref<Cell>> root, int mode = 0);
td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash,
td::FileFd& fd, int mode = 0);
td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash, td::FileFd& fd,
int mode = 0, td::CancellationToken cancellation_token = {});
} // namespace vm

14
crypto/vm/large-boc-serializer.cpp
View file

@ -33,7 +33,8 @@ class LargeBocSerializer {
public:
using Hash = Cell::Hash;
explicit LargeBocSerializer(std::shared_ptr<CellDbReader> reader) : reader(std::move(reader)) {
explicit LargeBocSerializer(std::shared_ptr<CellDbReader> reader, td::CancellationToken cancellation_token = {})
: reader(std::move(reader)), cancellation_token(std::move(cancellation_token)) {
}
void add_root(Hash root);
@ -84,6 +85,7 @@ class LargeBocSerializer {
int revisit(int cell_idx, int force = 0);
td::uint64 compute_sizes(int mode, int& r_size, int& o_size);
td::CancellationToken cancellation_token;
td::Timestamp log_speed_at_;
size_t processed_cells_ = 0;
static constexpr double LOG_SPEED_PERIOD = 120.0;
@ -112,6 +114,9 @@ td::Result<int> LargeBocSerializer::import_cell(Hash hash, int depth) {
return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large");
}
++processed_cells_;
if (processed_cells_ % 1000 == 0) {
TRY_STATUS(cancellation_token.check());
}
if (log_speed_at_.is_in_past()) {
log_speed_at_ += LOG_SPEED_PERIOD;
LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s";
@ -408,6 +413,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) {
store_ref(k);
}
++processed_cells_;
if (processed_cells_ % 1000 == 0) {
TRY_STATUS(cancellation_token.check());
}
if (log_speed_at_.is_in_past()) {
log_speed_at_ += LOG_SPEED_PERIOD;
LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s";
@ -428,10 +436,10 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) {
} // namespace
td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash, td::FileFd& fd,
int mode) {
int mode, td::CancellationToken cancellation_token) {
td::Timer timer;
CHECK(reader != nullptr)
LargeBocSerializer serializer(reader);
LargeBocSerializer serializer(reader, std::move(cancellation_token));
serializer.add_root(root_hash);
TRY_STATUS(serializer.import_cells());
TRY_STATUS(serializer.serialize(fd, mode));

2
dht-server/dht-server.cpp
View file

@ -172,7 +172,7 @@ ton::tl_object_ptr<ton::ton_api::engine_validator_config> Config::tl() const {
return ton::create_tl_object<ton::ton_api::engine_validator_config>(
out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec),
ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec),
nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec));
nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec));
}
td::Result<bool> Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip,

7
tdutils/td/utils/CancellationToken.h
View file

@ -20,6 +20,7 @@
#include <atomic>
#include <memory>
#include "Status.h"
namespace td {
@ -38,6 +39,12 @@ class CancellationToken {
}
return token_->is_cancelled_.load(std::memory_order_acquire);
}
Status check() const {
if (*this) {
return Status::Error(653, "cancelled"); // cancelled = 653
}
return Status::OK();
}
CancellationToken() = default;
explicit CancellationToken(std::shared_ptr<detail::RawCancellationToken> token) : token_(std::move(token)) {
}

5
tl/generate/scheme/ton_api.tl
View file

@ -609,13 +609,14 @@ engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config;
engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster;
engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave;
engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig;
engine.validator.extraConfig state_serializer_enabled:Bool = engine.validator.ExtraConfig;
engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl)
dht:(vector engine.dht)
validators:(vector engine.validator) collators:(vector engine.collator)
fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave)
fullnodemasters:(vector engine.validator.fullNodeMaster)
fullnodeconfig:engine.validator.fullNodeConfig
extraconfig:engine.validator.extraConfig
liteservers:(vector engine.liteServer) control:(vector engine.controlInterface)
shards_to_monitor:(vector tonNode.shardId)
gc:engine.gc = engine.validator.Config;
@ -741,6 +742,8 @@ engine.validator.addCustomOverlay overlay:engine.validator.customOverlay = engin
engine.validator.delCustomOverlay name:string = engine.validator.Success;
engine.validator.showCustomOverlays = engine.validator.CustomOverlaysConfig;
engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Success;
engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo;
engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success;

BIN
tl/generate/scheme/ton_api.tlo
View file

Binary file not shown.

23
validator-engine-console/validator-engine-console-query.cpp
View file

@ -1192,6 +1192,29 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) {
return td::Status::OK();
}
td::Status SetStateSerializerEnabledQuery::run() {
TRY_RESULT(value, tokenizer_.get_token<int>());
if (value != 0 && value != 1) {
return td::Status::Error("expected 0 or 1");
}
TRY_STATUS(tokenizer_.check_endl());
enabled_ = value;
return td::Status::OK();
}
td::Status SetStateSerializerEnabledQuery::send() {
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_setStateSerializerEnabled>(enabled_);
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
return td::Status::OK();
}
td::Status SetStateSerializerEnabledQuery::receive(td::BufferSlice data) {
TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
"received incorrect answer: ");
td::TerminalIO::out() << "success\n";
return td::Status::OK();
}
td::Status GetValidatorSessionsInfoQuery::run() {
TRY_STATUS(tokenizer_.check_endl());
return td::Status::OK();

22
validator-engine-console/validator-engine-console-query.h
View file

@ -1208,6 +1208,28 @@ class ShowCustomOverlaysQuery : public Query {
}
};
class SetStateSerializerEnabledQuery : public Query {
public:
SetStateSerializerEnabledQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
: Query(console, std::move(tokenizer)) {
}
td::Status run() override;
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "setstateserializerenabled";
}
static std::string get_help() {
return "setstateserializerenabled <value>\tdisable or enable persistent state serializer; value is 0 or 1";
}
std::string name() const override {
return get_name();
}
private:
bool enabled_;
};
class GetValidatorSessionsInfoQuery : public Query {
public:
GetValidatorSessionsInfoQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)

1
validator-engine-console/validator-engine-console.cpp
View file

@ -146,6 +146,7 @@ void ValidatorEngineConsole::run() {
add_query_runner(std::make_unique<QueryRunnerImpl<AddCustomOverlayQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<DelCustomOverlayQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<ShowCustomOverlaysQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<SetStateSerializerEnabledQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<GetValidatorSessionsInfoQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<AddCollatorQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<AddShardQuery>>());

61
validator-engine/validator-engine.cpp
View file

@ -73,6 +73,7 @@
#include "block-parse.h"
#include "common/delay.h"
#include "block/precompiled-smc/PrecompiledSmartContract.h"
#include "interfaces/validator-manager.h"
Config::Config() {
out_port = 3278;
@ -160,6 +161,11 @@ Config::Config(const ton::ton_api::engine_validator_config &config) {
if (config.fullnodeconfig_) {
full_node_config = ton::validator::fullnode::FullNodeConfig(config.fullnodeconfig_);
}
if (config.extraconfig_) {
state_serializer_enabled = config.extraconfig_->state_serializer_enabled_;
} else {
state_serializer_enabled = true;
}
for (auto &serv : config.liteservers_) {
config_add_lite_server(ton::PublicKeyHash{serv->id_}, serv->port_).ensure();
@ -245,6 +251,12 @@ ton::tl_object_ptr<ton::ton_api::engine_validator_config> Config::tl() const {
full_node_config_obj = full_node_config.tl();
}
ton::tl_object_ptr<ton::ton_api::engine_validator_extraConfig> extra_config_obj = {};
if (!state_serializer_enabled) {
// Non-default values
extra_config_obj = ton::create_tl_object<ton::ton_api::engine_validator_extraConfig>(state_serializer_enabled);
}
std::vector<ton::tl_object_ptr<ton::ton_api::engine_liteServer>> liteserver_vec;
for (auto &x : liteservers) {
liteserver_vec.push_back(ton::create_tl_object<ton::ton_api::engine_liteServer>(x.second.tl(), x.first));
@ -273,8 +285,8 @@ ton::tl_object_ptr<ton::ton_api::engine_validator_config> Config::tl() const {
return ton::create_tl_object<ton::ton_api::engine_validator_config>(
out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec),
full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec),
std::move(full_node_config_obj), std::move(liteserver_vec), std::move(control_vec), std::move(shards_vec),
std::move(gc_vec));
std::move(full_node_config_obj), std::move(extra_config_obj), std::move(liteserver_vec), std::move(control_vec),
std::move(shards_vec), std::move(gc_vec));
}
td::Result<bool> Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip,
@ -1459,6 +1471,7 @@ td::Status ValidatorEngine::load_global_config() {
h.push_back(b);
}
validator_options_.write().set_hardforks(std::move(h));
validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled);
validator_options_.write().set_validator_mode(validator_mode_);
return td::Status::OK();
@ -3751,6 +3764,34 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCusto
custom_overlays_config_, true));
}
void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query,
td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
td::Promise<td::BufferSlice> promise) {
if (!(perm & ValidatorEnginePermissions::vep_modify)) {
promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
return;
}
if (!started_) {
promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started")));
return;
}
if (query.enabled_ == validator_options_->get_state_serializer_enabled()) {
promise.set_value(ton::create_serialize_tl_object<ton::ton_api::engine_validator_success>());
return;
}
validator_options_.write().set_state_serializer_enabled(query.enabled_);
td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options,
validator_options_);
config_.state_serializer_enabled = query.enabled_;
write_config([promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
promise.set_value(create_control_query_error(R.move_as_error()));
} else {
promise.set_value(ton::create_serialize_tl_object<ton::ton_api::engine_validator_success>());
}
});
}
void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query,
td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
td::Promise<td::BufferSlice> promise) {
@ -4150,7 +4191,7 @@ int main(int argc, char *argv[]) {
acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); });
return td::Status::OK();
});
p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400",
p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=86400",
[&](td::Slice fname) {
auto v = td::to_double(fname);
if (v <= 0) {
@ -4160,7 +4201,7 @@ int main(int argc, char *argv[]) {
return td::Status::OK();
});
p.add_checked_option(
'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400",
'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=7*86400",
[&](td::Slice fname) {
auto v = td::to_double(fname);
if (v <= 0) {
@ -4290,7 +4331,7 @@ int main(int argc, char *argv[]) {
acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); });
});
p.add_checked_option(
'\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 50G)",
'\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)",
[&](td::Slice s) -> td::Status {
TRY_RESULT(v, td::to_integer_safe<td::uint64>(s));
if (v == 0) {
@ -4300,12 +4341,12 @@ int main(int argc, char *argv[]) {
return td::Status::OK();
});
p.add_option(
'\0', "celldb-no-direct-io", "disable direct I/O mode for RocksDb in CellDb (forced when celldb cache is < 30G)",
[&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, false); }); });
'\0', "celldb-direct-io", "enable direct I/O mode for RocksDb in CellDb (doesn't apply when celldb cache is < 30G)",
[&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, true); }); });
p.add_option(
'\0', "celldb-no-preload-all",
"disable preloading all cells from CellDb on startup (enabled by default)",
[&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, false); }); });
'\0', "celldb-preload-all",
"preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and celldb-direct-io)",
[&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); });
p.add_checked_option(
'\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)",
[&](td::Slice s) -> td::Status {

10
validator-engine/validator-engine.hpp
View file

@ -100,6 +100,8 @@ struct Config {
std::set<ton::PublicKeyHash> gc;
std::vector<ton::ShardIdFull> shards_to_monitor;
bool state_serializer_enabled = true;
void decref(ton::PublicKeyHash key);
void incref(ton::PublicKeyHash key) {
keys_refcnt[key]++;
@ -223,9 +225,9 @@ class ValidatorEngine : public td::actor::Actor {
double archive_preload_period_ = 0.0;
bool disable_rocksdb_stats_ = false;
bool nonfinal_ls_queries_enabled_ = false;
td::optional<td::uint64> celldb_cache_size_ = 50LL << 30;
bool celldb_direct_io_ = true;
bool celldb_preload_all_ = true;
td::optional<td::uint64> celldb_cache_size_ = 1LL << 30;
bool celldb_direct_io_ = false;
bool celldb_preload_all_ = false;
td::optional<double> catchain_max_block_delay_;
bool read_config_ = false;
bool started_keyring_ = false;
@ -510,6 +512,8 @@ class ValidatorEngine : public td::actor::Actor {
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
void run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data,
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data,
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
template <class T>
void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
td::Promise<td::BufferSlice> promise) {

9
validator/full-node-private-overlay.hpp
View file

@ -47,15 +47,11 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor {
config_ = std::move(config);
}
void set_enable_compression(bool value) {
enable_compression_ = value;
}
void start_up() override;
void tear_down() override;
FullNodePrivateBlockOverlay(adnl::AdnlNodeIdShort local_id, std::vector<adnl::AdnlNodeIdShort> nodes,
FileHash zero_state_file_hash, FullNodeConfig config, bool enable_compression,
FileHash zero_state_file_hash, FullNodeConfig config,
td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2,
td::actor::ActorId<overlay::Overlays> overlays,
@ -65,7 +61,6 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor {
, nodes_(std::move(nodes))
, zero_state_file_hash_(zero_state_file_hash)
, config_(config)
, enable_compression_(enable_compression)
, keyring_(keyring)
, adnl_(adnl)
, rldp_(rldp)
@ -80,7 +75,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor {
std::vector<adnl::AdnlNodeIdShort> nodes_;
FileHash zero_state_file_hash_;
FullNodeConfig config_;
bool enable_compression_;
bool enable_compression_ = true;
td::actor::ActorId<keyring::Keyring> keyring_;
td::actor::ActorId<adnl::Adnl> adnl_;

64
validator/full-node.cpp
View file

@ -531,14 +531,12 @@ td::actor::ActorId<FullNodeShard> FullNodeImpl::get_shard(AccountIdPrefixFull ds
return get_shard(shard_prefix(dst, max_shard_pfx_len));
}
void FullNodeImpl::got_key_block_state(td::Ref<ShardState> state) {
auto m = td::Ref<MasterchainState>{std::move(state)};
void FullNodeImpl::got_key_block_config(td::Ref<ConfigHolder> config) {
PublicKeyHash l = PublicKeyHash::zero();
std::vector<PublicKeyHash> keys;
std::map<PublicKeyHash, adnl::AdnlNodeIdShort> current_validators;
for (td::int32 i = -1; i <= 1; i++) {
auto r = m->get_total_validator_set(i < 0 ? i : 1 - i);
auto r = config->get_total_validator_set(i < 0 ? i : 1 - i);
if (r.not_null()) {
auto vec = r->export_vector();
for (auto &el : vec) {
@ -554,16 +552,15 @@ void FullNodeImpl::got_key_block_state(td::Ref<ShardState> state) {
}
}
set_private_block_overlays_enable_compression(m->get_consensus_config().proto_version >= 3);
if (current_validators != current_validators_) {
current_validators_ = std::move(current_validators);
update_private_overlays();
}
if (keys == all_validators_) {
return;
}
// Let's turn off this optimization, since keyblocks are rare enough to update on each keyblock
// if (keys == all_validators_) {
// return;
// }
all_validators_ = keys;
sign_cert_by_ = l;
@ -577,15 +574,31 @@ void FullNodeImpl::got_key_block_state(td::Ref<ShardState> state) {
}
void FullNodeImpl::new_key_block(BlockHandle handle) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
if (R.is_error()) {
VLOG(FULL_NODE_WARNING) << "failed to get key block state: " << R.move_as_error();
} else {
td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_state, R.move_as_ok());
}
});
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle,
std::move(P));
if (handle->id().seqno() == 0) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
if (R.is_error()) {
VLOG(FULL_NODE_WARNING) << "failed to get zero state: " << R.move_as_error();
} else {
auto s = td::Ref<MasterchainState>{R.move_as_ok()};
CHECK(s.not_null());
td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config, s->get_config_holder().move_as_ok());
}
});
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle,
std::move(P));
} else {
CHECK(handle->is_key_block());
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ProofLink>> R) {
if (R.is_error()) {
VLOG(FULL_NODE_WARNING) << "failed to get key block proof: " << R.move_as_error();
} else {
td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config,
R.ok()->get_key_block_config().move_as_ok());
}
});
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_proof_link_from_db, handle,
std::move(P));
}
}
void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) {
@ -717,16 +730,6 @@ void FullNodeImpl::update_private_overlays() {
}
}
void FullNodeImpl::set_private_block_overlays_enable_compression(bool value) {
if (private_block_overlays_enable_compression_ == value) {
return;
}
private_block_overlays_enable_compression_ = true;
for (auto &p : private_block_overlays_) {
td::actor::send_closure(p.second, &FullNodePrivateBlockOverlay::set_enable_compression, value);
}
}
void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) {
if (!use_old_private_overlays_) {
return;
@ -738,9 +741,8 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) {
nodes.push_back(p.second);
}
private_block_overlays_[key] = td::actor::create_actor<FullNodePrivateBlockOverlay>(
"BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_,
private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_,
actor_id(this));
"BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_,
adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this));
}
}

4
validator/full-node.hpp
View file

@ -88,7 +88,7 @@ class FullNodeImpl : public FullNode {
block::ImportedMsgQueueLimits limits, td::Timestamp timeout,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise);
void got_key_block_state(td::Ref<ShardState> state);
void got_key_block_config(td::Ref<ConfigHolder> config);
void new_key_block(BlockHandle handle);
void process_block_broadcast(BlockBroadcast broadcast) override;
@ -149,7 +149,6 @@ class FullNodeImpl : public FullNode {
// New overlays (v2) - overlay per shard (monitor_min_split depth).
bool use_old_private_overlays_ = false; // TODO: set from config
std::map<PublicKeyHash, td::actor::ActorOwn<FullNodePrivateBlockOverlay>> private_block_overlays_;
bool private_block_overlays_enable_compression_ = false;
bool broadcast_block_candidates_in_public_overlay_ = false;
FullNodePrivateBlockOverlaysV2 private_block_overlays_v2_;
@ -162,7 +161,6 @@ class FullNodeImpl : public FullNode {
std::queue<BlockIdExt> custom_overlays_sent_broadcasts_lru_;
void update_private_overlays();
void set_private_block_overlays_enable_compression(bool value);
void create_private_block_overlay(PublicKeyHash key);
void update_custom_overlay(CustomOverlayInfo& overlay);
void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast);

6
validator/manager-hardfork.hpp
View file

@ -493,6 +493,9 @@ class ValidatorManagerImpl : public ValidatorManager {
td::Promise<tl_object_ptr<lite_api::liteServer_nonfinal_validatorGroups>> promise) override {
promise.set_result(td::Status::Error("not implemented"));
}
void update_options(td::Ref<ValidatorManagerOptions> opts) override {
opts_ = std::move(opts);
}
void validated_new_block(BlockIdExt block_id) override {
}
void add_persistent_state_description(td::Ref<PersistentStateDescription> desc) override {
@ -508,9 +511,6 @@ class ValidatorManagerImpl : public ValidatorManager {
void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override {
UNREACHABLE();
}
void update_options(td::Ref<ValidatorManagerOptions> opts) override {
opts_ = std::move(opts);
}
private:
td::Ref<ValidatorManagerOptions> opts_;

32
validator/manager.cpp
View file

@ -3334,6 +3334,22 @@ void ValidatorManagerImpl::get_validator_groups_info_for_litequery(
td::actor::create_actor<Actor>("get-validator-groups-info", std::move(groups), std::move(promise)).release();
}
void ValidatorManagerImpl::update_options(td::Ref<ValidatorManagerOptions> opts) {
if (!shard_client_.empty()) {
td::actor::send_closure(shard_client_, &ShardClient::update_options, opts);
}
if (!serializer_.empty()) {
td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts);
}
if (!out_msg_queue_importer_.empty()) {
td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::update_options, opts);
}
if (!queue_size_counter_.empty()) {
td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::update_options, opts);
}
opts_ = std::move(opts);
}
void ValidatorManagerImpl::get_validator_sessions_info(
td::Promise<tl_object_ptr<ton_api::engine_validator_validatorSessionsInfo>> promise) {
std::vector<td::actor::ActorId<ValidatorGroup>> groups;
@ -3400,22 +3416,6 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh
}
}
void ValidatorManagerImpl::update_options(td::Ref<ValidatorManagerOptions> opts) {
if (!shard_client_.empty()) {
td::actor::send_closure(shard_client_, &ShardClient::update_options, opts);
}
if (!serializer_.empty()) {
td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts);
}
if (!out_msg_queue_importer_.empty()) {
td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::update_options, opts);
}
if (!queue_size_counter_.empty()) {
td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::update_options, opts);
}
opts_ = std::move(opts);
}
void ValidatorManagerImpl::add_persistent_state_description(td::Ref<PersistentStateDescription> desc) {
auto now = (UnixTime)td::Clocks::system();
if (desc->end_time <= now) {

3
validator/manager.hpp
View file

@ -615,6 +615,8 @@ class ValidatorManagerImpl : public ValidatorManager {
void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override;
void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override;
void update_options(td::Ref<ValidatorManagerOptions> opts) override;
void get_validator_sessions_info(
td::Promise<tl_object_ptr<ton_api::engine_validator_validatorSessionsInfo>> promise) override;
@ -627,7 +629,6 @@ class ValidatorManagerImpl : public ValidatorManager {
void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override;
void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override;
void update_options(td::Ref<ValidatorManagerOptions> opts) override;
void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint32> promise) override {
if (last_masterchain_state_.is_null()) {

47
validator/state-serializer.cpp
View file

@ -26,6 +26,9 @@ namespace ton {
namespace validator {
void AsyncStateSerializer::start_up() {
if (!opts_->get_state_serializer_enabled()) {
LOG(ERROR) << "Persistent state serializer is disabled";
}
alarm_timestamp() = td::Timestamp::in(1.0 + td::Random::fast(0, 10) * 1.0);
running_ = true;
@ -129,7 +132,7 @@ void AsyncStateSerializer::next_iteration() {
}
CHECK(masterchain_handle_->id() == last_block_id_);
if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno &&
need_serialize(masterchain_handle_)) {
need_serialize(masterchain_handle_) && opts_->get_state_serializer_enabled()) {
if (!stored_persistent_state_description_) {
LOG(INFO) << "storing persistent state description for " << masterchain_handle_->id().id;
running_ = true;
@ -184,6 +187,9 @@ void AsyncStateSerializer::next_iteration() {
return;
}
if (masterchain_handle_->inited_next_left()) {
if (need_serialize(masterchain_handle_) && !opts_->get_state_serializer_enabled()) {
LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str();
}
last_block_id_ = masterchain_handle_->one_next(true);
have_masterchain_state_ = false;
stored_persistent_state_description_ = false;
@ -229,6 +235,10 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) {
void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state,
std::shared_ptr<vm::CellDbReader> cell_db_reader) {
if (!opts_->get_state_serializer_enabled()) {
stored_masterchain_state();
return;
}
LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str();
have_masterchain_state_ = true;
CHECK(next_idx_ == 0);
@ -241,11 +251,16 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state
}
}
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) {
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31);
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader,
cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable {
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token));
};
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
R.ensure();
if (R.is_error() && R.error().code() == cancelled) {
LOG(ERROR) << "Persistent state serialization cancelled";
} else {
R.ensure();
}
td::actor::send_closure(SelfId, &AsyncStateSerializer::stored_masterchain_state);
});
@ -284,13 +299,22 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) {
void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardState> state,
std::shared_ptr<vm::CellDbReader> cell_db_reader) {
if (!opts_->get_state_serializer_enabled()) {
success_handler();
return;
}
LOG(ERROR) << "serializing shard state " << handle->id().id.to_str();
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) {
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31);
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader,
cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable {
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token));
};
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Unit> R) {
R.ensure();
LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str();
if (R.is_error() && R.error().code() == cancelled) {
LOG(ERROR) << "Persistent state serialization cancelled";
} else {
R.ensure();
LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str();
}
td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler);
});
td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(),
@ -316,6 +340,13 @@ void AsyncStateSerializer::success_handler() {
next_iteration();
}
void AsyncStateSerializer::update_options(td::Ref<ValidatorManagerOptions> opts) {
opts_ = std::move(opts);
if (!opts_->get_state_serializer_enabled()) {
cancellation_token_source_.cancel();
}
}
bool AsyncStateSerializer::need_serialize(BlockHandle handle) {
if (handle->id().id.seqno == 0 || !handle->is_key_block()) {
return false;

5
validator/state-serializer.hpp
View file

@ -37,6 +37,7 @@ class AsyncStateSerializer : public td::actor::Actor {
bool saved_to_db_ = true;
td::Ref<ValidatorManagerOptions> opts_;
td::CancellationTokenSource cancellation_token_source_;
td::actor::ActorId<ValidatorManager> manager_;
@ -91,9 +92,7 @@ class AsyncStateSerializer : public td::actor::Actor {
void fail_handler_cont();
void success_handler();
void update_options(td::Ref<ValidatorManagerOptions> opts) {
opts_ = std::move(opts);
}
void update_options(td::Ref<ValidatorManagerOptions> opts);
};
} // namespace validator

7
validator/validator-options.hpp
View file

@ -139,6 +139,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
td::optional<double> get_catchain_max_block_delay() const override {
return catchain_max_block_delay_;
}
bool get_state_serializer_enabled() const override {
return state_serializer_enabled_;
}
ValidatorMode validator_mode() const override {
return validator_mode_;
}
@ -222,6 +225,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
void set_catchain_max_block_delay(double value) override {
catchain_max_block_delay_ = value;
}
void set_state_serializer_enabled(bool value) override {
state_serializer_enabled_ = value;
}
void set_validator_mode(ValidatorMode value) override {
validator_mode_ = value;
}
@ -274,6 +280,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
bool celldb_direct_io_ = false;
bool celldb_preload_all_ = false;
td::optional<double> catchain_max_block_delay_;
bool state_serializer_enabled_ = true;
ValidatorMode validator_mode_ = validator_normal;
};

12
validator/validator.h
View file

@ -90,6 +90,7 @@ struct ValidatorManagerOptions : public td::CntObject {
virtual bool get_celldb_direct_io() const = 0;
virtual bool get_celldb_preload_all() const = 0;
virtual td::optional<double> get_catchain_max_block_delay() const = 0;
virtual bool get_state_serializer_enabled() const = 0;
virtual ValidatorMode validator_mode() const = 0;
virtual void set_zero_block_id(BlockIdExt block_id) = 0;
@ -118,15 +119,16 @@ struct ValidatorManagerOptions : public td::CntObject {
virtual void set_celldb_direct_io(bool value) = 0;
virtual void set_celldb_preload_all(bool value) = 0;
virtual void set_catchain_max_block_delay(double value) = 0;
virtual void set_state_serializer_enabled(bool value) = 0;
virtual void set_validator_mode(ValidatorMode value) = 0;
static td::Ref<ValidatorManagerOptions> create(
BlockIdExt zero_block_id, BlockIdExt init_block_id,
std::function<bool(ShardIdFull)> check_shard = [](ShardIdFull) { return true; },
bool allow_blockchain_init = false, double sync_blocks_before = 86400, double block_ttl = 86400 * 7,
double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650,
double max_mempool_num = 999999,
bool initial_sync_disabled = false);
bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400,
double state_ttl = 3600, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650,
double max_mempool_num = 999999, bool initial_sync_disabled = false);
};
class ValidatorManagerInterface : public td::actor::Actor {
@ -262,13 +264,13 @@ class ValidatorManagerInterface : public td::actor::Actor {
virtual void add_perf_timer_stat(std::string name, double duration) = 0;
virtual void get_out_msg_queue_size(BlockIdExt block_id, td::Promise<td::uint32> promise) = 0;
virtual void update_options(td::Ref<ValidatorManagerOptions> opts) = 0;
virtual void get_validator_sessions_info(
td::Promise<tl_object_ptr<ton_api::engine_validator_validatorSessionsInfo>> promise) = 0;
virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0;
virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0;
virtual void update_options(td::Ref<ValidatorManagerOptions> opts) = 0;
};
} // namespace validator