external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Merge branch 'safe_features' into testnet-update

Browse source
This commit is contained in:
SpyCheese 2024-09-06 11:48:46 +03:00
commit feff73c4be
41 changed files with 1066 additions and 207 deletions
Download patch file Download diff file Expand all files Collapse all files

13
validator/db/celldb.cpp
View file

@ -188,12 +188,17 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promi
if (!opts_->get_disable_rocksdb_stats()) {
cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6);
}
LOG(DEBUG) << "Stored state " << block_id.to_str();
}
void CellDbIn::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) {
promise.set_result(boc_->get_cell_db_reader());
}
void CellDbIn::get_last_deleted_mc_state(td::Promise<BlockSeqno> promise) {
promise.set_result(last_deleted_mc_state_);
}
void CellDbIn::flush_db_stats() {
if (opts_->get_disable_rocksdb_stats()) {
return;
@ -320,6 +325,10 @@ void CellDbIn::gc_cont2(BlockHandle handle) {
if (!opts_->get_disable_rocksdb_stats()) {
cell_db_statistics_.gc_cell_time_.insert(timer.elapsed() * 1e6);
}
if (handle->id().is_masterchain()) {
last_deleted_mc_state_ = handle->id().seqno();
}
LOG(DEBUG) << "Deleted state " << handle->id().to_str();
}
void CellDbIn::skip_gc() {
@ -453,6 +462,10 @@ void CellDb::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> p
td::actor::send_closure(cell_db_, &CellDbIn::get_cell_db_reader, std::move(promise));
}
void CellDb::get_last_deleted_mc_state(td::Promise<BlockSeqno> promise) {
td::actor::send_closure(cell_db_, &CellDbIn::get_last_deleted_mc_state, std::move(promise));
}
void CellDb::start_up() {
CellDbBase::start_up();
boc_ = vm::DynamicBagOfCellsDb::create();

3
validator/db/celldb.hpp
View file

@ -61,6 +61,7 @@ class CellDbIn : public CellDbBase {
void load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise);
void store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise);
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise);
void get_last_deleted_mc_state(td::Promise<BlockSeqno> promise);
void migrate_cell(td::Bits256 hash);
@ -143,6 +144,7 @@ class CellDbIn : public CellDbBase {
std::shared_ptr<td::RocksDbSnapshotStatistics> snapshot_statistics_;
CellDbStatistics cell_db_statistics_;
td::Timestamp statistics_flush_at_ = td::Timestamp::never();
BlockSeqno last_deleted_mc_state_ = 0;
public:
class MigrationProxy : public td::actor::Actor {
@ -167,6 +169,7 @@ class CellDb : public CellDbBase {
boc_->set_loader(std::make_unique<vm::CellLoader>(std::move(snapshot), on_load_callback_)).ensure();
}
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise);
void get_last_deleted_mc_state(td::Promise<BlockSeqno> promise);
CellDb(td::actor::ActorId<RootDb> root_db, std::string path, td::Ref<ValidatorManagerOptions> opts)
: root_db_(root_db), path_(path), opts_(opts) {

4
validator/db/rootdb.cpp
View file

@ -285,6 +285,10 @@ void RootDb::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> p
td::actor::send_closure(cell_db_, &CellDb::get_cell_db_reader, std::move(promise));
}
void RootDb::get_last_deleted_mc_state(td::Promise<BlockSeqno> promise) {
td::actor::send_closure(cell_db_, &CellDb::get_last_deleted_mc_state, std::move(promise));
}
void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
td::Promise<td::Unit> promise) {
td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id,

1
validator/db/rootdb.hpp
View file

@ -64,6 +64,7 @@ class RootDb : public Db {
td::Promise<td::Ref<ShardState>> promise) override;
void get_block_state(ConstBlockHandle handle, td::Promise<td::Ref<ShardState>> promise) override;
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) override;
void get_last_deleted_mc_state(td::Promise<BlockSeqno> promise) override;
void store_block_handle(BlockHandle handle, td::Promise<td::Unit> promise) override;
void get_block_handle(BlockIdExt id, td::Promise<BlockHandle> promise) override;

32
validator/impl/collator.cpp
View file

@ -3064,7 +3064,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R
bool defer = false;
if (!from_dispatch_queue) {
if (deferring_messages_enabled_ && collator_opts_->deferring_enabled && !is_special && !is_special_account &&
msg.msg_idx != 0) {
!collator_opts_->whitelist.count({src_wc, src_addr}) && msg.msg_idx != 0) {
if (++sender_generated_messages_count_[src_addr] >= collator_opts_->defer_messages_after ||
out_msg_queue_size_ > defer_out_queue_size_limit_) {
defer = true;
@ -3695,6 +3695,8 @@ bool Collator::process_dispatch_queue() {
vm::AugmentedDictionary cur_dispatch_queue{dispatch_queue_->get_root(), 256, block::tlb::aug_DispatchQueue};
std::map<std::tuple<WorkchainId, StdSmcAddress, LogicalTime>, size_t> count_per_initiator;
size_t total_count = 0;
auto prioritylist = collator_opts_->prioritylist;
auto prioritylist_iter = prioritylist.begin();
while (!cur_dispatch_queue.is_empty()) {
block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal);
if (block_full_) {
@ -3711,9 +3713,30 @@ bool Collator::process_dispatch_queue() {
return true;
}
StdSmcAddress src_addr;
auto account_dispatch_queue = block::get_dispatch_queue_min_lt_account(cur_dispatch_queue, src_addr);
td::Ref<vm::CellSlice> account_dispatch_queue;
while (!prioritylist.empty()) {
if (prioritylist_iter == prioritylist.end()) {
prioritylist_iter = prioritylist.begin();
}
auto priority_addr = *prioritylist_iter;
if (priority_addr.first != workchain() || !is_our_address(priority_addr.second)) {
prioritylist_iter = prioritylist.erase(prioritylist_iter);
continue;
}
src_addr = priority_addr.second;
account_dispatch_queue = cur_dispatch_queue.lookup(src_addr);
if (account_dispatch_queue.is_null()) {
prioritylist_iter = prioritylist.erase(prioritylist_iter);
} else {
++prioritylist_iter;
break;
}
}
if (account_dispatch_queue.is_null()) {
return fatal_error("invalid dispatch queue in shard state");
account_dispatch_queue = block::get_dispatch_queue_min_lt_account(cur_dispatch_queue, src_addr);
if (account_dispatch_queue.is_null()) {
return fatal_error("invalid dispatch queue in shard state");
}
}
vm::Dictionary dict{64};
td::uint64 dict_size;
@ -3733,7 +3756,8 @@ bool Collator::process_dispatch_queue() {
// Remove message from DispatchQueue
bool ok;
if (iter == 0 ||
(iter == 1 && sender_generated_messages_count_[src_addr] >= collator_opts_->defer_messages_after)) {
(iter == 1 && sender_generated_messages_count_[src_addr] >= collator_opts_->defer_messages_after &&
!collator_opts_->whitelist.count({workchain(), src_addr}))) {
ok = cur_dispatch_queue.lookup_delete(src_addr).not_null();
} else {
dict.lookup_delete(key);

2
validator/impl/liteserver.cpp
View file

@ -870,7 +870,7 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St
vm::FakeVmStateLimits fstate(1000); // limit recursive (de)serialization calls
vm::VmStateInterface::Guard guard(&fstate);
auto cs = vm::load_cell_slice(res.move_as_ok());
if (!(vm::Stack::deserialize_to(cs, stack_, 0) && cs.empty_ext())) {
if (!(vm::Stack::deserialize_to(cs, stack_, 2 /* no continuations */) && cs.empty_ext())) {
fatal_error("parameter list boc cannot be deserialized as a VmStack");
return;
}

2
validator/impl/shard.cpp
View file

@ -44,6 +44,7 @@ ShardStateQ::ShardStateQ(const ShardStateQ& other)
, root(other.root)
, lt(other.lt)
, utime(other.utime)
, global_id_(other.global_id_)
, before_split_(other.before_split_)
, fake_split_(other.fake_split_)
, fake_merge_(other.fake_merge_) {
@ -121,6 +122,7 @@ td::Status ShardStateQ::init() {
}
lt = info.gen_lt;
utime = info.gen_utime;
global_id_ = info.global_id;
before_split_ = info.before_split;
block::ShardId id{info.shard_id};
ton::BlockId hdr_id{ton::ShardIdFull(id), info.seq_no};

4
validator/impl/shard.hpp
View file

@ -38,6 +38,7 @@ class ShardStateQ : virtual public ShardState {
Ref<vm::Cell> root;
LogicalTime lt{0};
UnixTime utime{0};
td::int32 global_id_{0};
bool before_split_{false};
bool fake_split_{false};
bool fake_merge_{false};
@ -81,6 +82,9 @@ class ShardStateQ : virtual public ShardState {
LogicalTime get_logical_time() const override {
return lt;
}
td::int32 get_global_id() const override {
return global_id_;
}
td::optional<BlockIdExt> get_master_ref() const override {
return master_ref;
}

1
validator/interfaces/db.h
View file

@ -52,6 +52,7 @@ class Db : public td::actor::Actor {
td::Promise<td::Ref<ShardState>> promise) = 0;
virtual void get_block_state(ConstBlockHandle handle, td::Promise<td::Ref<ShardState>> promise) = 0;
virtual void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) = 0;
virtual void get_last_deleted_mc_state(td::Promise<BlockSeqno> promise) = 0;
virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
td::Promise<td::Unit> promise) = 0;

1
validator/interfaces/shard.h
View file

@ -39,6 +39,7 @@ class ShardState : public td::CntObject {
virtual UnixTime get_unix_time() const = 0;
virtual LogicalTime get_logical_time() const = 0;
virtual td::int32 get_global_id() const = 0;
virtual ShardIdFull get_shard() const = 0;
virtual BlockSeqno get_seqno() const = 0;
virtual BlockIdExt get_block_id() const = 0;

25
validator/manager.cpp
View file

@ -2186,7 +2186,13 @@ void ValidatorManagerImpl::update_shards() {
last_masterchain_state_->get_total_validator_set(0)->export_vector().size());
}
}
} // namespace validator
if (!serializer_.empty()) {
td::actor::send_closure(
serializer_, &AsyncStateSerializer::auto_disable_serializer,
!validator_groups_.empty() && last_masterchain_state_->get_global_id() == -239); // mainnet only
}
}
void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vector<td::actor::ActorId<ValidatorGroup>> list) {
for (auto &v : list) {
@ -2779,6 +2785,23 @@ void ValidatorManagerImpl::prepare_stats(td::Promise<std::vector<std::pair<std::
vec.emplace_back("rotatemasterchainblock", last_rotate_block_id_.to_str());
//vec.emplace_back("shardclientmasterchainseqno", td::to_string(min_confirmed_masterchain_seqno_));
vec.emplace_back("stateserializermasterchainseqno", td::to_string(state_serializer_masterchain_seqno_));
td::actor::send_closure(db_, &Db::get_last_deleted_mc_state,
[promise = merger.make_promise(""),
gc_seqno = gc_masterchain_handle_->id().seqno()](td::Result<BlockSeqno> R) mutable {
TRY_RESULT_PROMISE(promise, seqno, std::move(R));
std::string s;
if (seqno == 0) {
s = "none";
} else if (seqno <= gc_seqno) {
s = PSTRING() << seqno << " (gc_seqno-" << (gc_seqno - seqno) << ")";
} else {
s = PSTRING() << seqno << " (gc_seqno+" << (seqno - gc_seqno) << ")";
}
std::vector<std::pair<std::string, std::string>> vec;
vec.emplace_back("lastgcdmasterchainstate", std::move(s));
promise.set_value(std::move(vec));
});
}
if (!shard_client_.empty()) {

2
validator/state-serializer.cpp
View file

@ -378,6 +378,7 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) {
void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardState> state,
std::shared_ptr<vm::CellDbReader> cell_db_reader) {
next_idx_++;
if (!opts_->get_state_serializer_enabled() || auto_disabled_) {
success_handler();
return;
@ -406,7 +407,6 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardStat
});
td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(),
masterchain_handle_->id(), write_data, std::move(P));
next_idx_++;
}
void AsyncStateSerializer::fail_handler(td::Status reason) {

10
validator/validator-group.cpp
View file

@ -348,10 +348,12 @@ void ValidatorGroup::create_session() {
<< ".",
allow_unsafe_self_blocks_resync_);
}
if (opts_->get_catchain_max_block_delay()) {
td::actor::send_closure(session_, &validatorsession::ValidatorSession::set_catchain_max_block_delay,
opts_->get_catchain_max_block_delay().value());
}
double catchain_delay = opts_->get_catchain_max_block_delay() ? opts_->get_catchain_max_block_delay().value() : 0.4;
double catchain_delay_slow =
std::max(catchain_delay,
opts_->get_catchain_max_block_delay_slow() ? opts_->get_catchain_max_block_delay_slow().value() : 1.0);
td::actor::send_closure(session_, &validatorsession::ValidatorSession::set_catchain_max_block_delay, catchain_delay,
catchain_delay_slow);
if (started_) {
td::actor::send_closure(session_, &validatorsession::ValidatorSession::start);
}

8
validator/validator-options.hpp
View file

@ -141,6 +141,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
td::optional<double> get_catchain_max_block_delay() const override {
return catchain_max_block_delay_;
}
td::optional<double> get_catchain_max_block_delay_slow() const override {
return catchain_max_block_delay_slow_;
}
bool get_state_serializer_enabled() const override {
return state_serializer_enabled_;
}
@ -230,6 +233,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
void set_catchain_max_block_delay(double value) override {
catchain_max_block_delay_ = value;
}
void set_catchain_max_block_delay_slow(double value) override {
catchain_max_block_delay_slow_ = value;
}
void set_state_serializer_enabled(bool value) override {
state_serializer_enabled_ = value;
}
@ -289,7 +295,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
td::optional<td::uint64> celldb_cache_size_;
bool celldb_direct_io_ = false;
bool celldb_preload_all_ = false;
td::optional<double> catchain_max_block_delay_;
td::optional<double> catchain_max_block_delay_, catchain_max_block_delay_slow_;
bool state_serializer_enabled_ = true;
td::Ref<CollatorOptions> collator_options_{true};
bool fast_state_serializer_enabled_ = false;

7
validator/validator.h
View file

@ -64,6 +64,11 @@ struct CollatorOptions : public td::CntObject {
td::uint32 dispatch_phase_3_max_total = 150;
td::uint32 dispatch_phase_2_max_per_initiator = 20;
td::optional<td::uint32> dispatch_phase_3_max_per_initiator; // Default - depends on out msg queue size
// Don't defer messages from these accounts
std::set<std::pair<WorkchainId, StdSmcAddress>> whitelist;
// Prioritize these accounts on each phase of process_dispatch_queue
std::set<std::pair<WorkchainId, StdSmcAddress>> prioritylist;
};
struct ValidatorManagerOptions : public td::CntObject {
@ -105,6 +110,7 @@ struct ValidatorManagerOptions : public td::CntObject {
virtual bool get_celldb_direct_io() const = 0;
virtual bool get_celldb_preload_all() const = 0;
virtual td::optional<double> get_catchain_max_block_delay() const = 0;
virtual td::optional<double> get_catchain_max_block_delay_slow() const = 0;
virtual bool get_state_serializer_enabled() const = 0;
virtual td::Ref<CollatorOptions> get_collator_options() const = 0;
virtual bool get_fast_state_serializer_enabled() const = 0;
@ -136,6 +142,7 @@ struct ValidatorManagerOptions : public td::CntObject {
virtual void set_celldb_direct_io(bool value) = 0;
virtual void set_celldb_preload_all(bool value) = 0;
virtual void set_catchain_max_block_delay(double value) = 0;
virtual void set_catchain_max_block_delay_slow(double value) = 0;
virtual void set_state_serializer_enabled(bool value) = 0;
virtual void set_collator_options(td::Ref<CollatorOptions> value) = 0;
virtual void set_fast_state_serializer_enabled(bool value) = 0;