external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Merge branch 'testnet' into accelerator

Browse source
This commit is contained in:
SpyCheese 2024-10-02 17:49:12 +03:00
commit 1c2c777644
21 changed files with 282 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

2
validator/fabric.h
View file

@ -89,7 +89,7 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc
Ed25519_PublicKey creator, td::Ref<ValidatorSet> validator_set,
td::Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager,
td::Timestamp timeout, td::Promise<BlockCandidate> promise,
td::CancellationToken cancellation_token = {}, unsigned mode = 0);
td::CancellationToken cancellation_token, unsigned mode, int attempt_idx = 0);
void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector<BlockIdExt> prev,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
td::Promise<BlockCandidate> promise);

5
validator/impl/collator-impl.h
View file

@ -77,6 +77,8 @@ class Collator final : public td::actor::Actor {
td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_;
td::Promise<BlockCandidate> main_promise;
unsigned mode_ = 0;
int attempt_idx_;
bool allow_repeat_collation_ = false;
ton::BlockSeqno last_block_seqno{0};
ton::BlockSeqno prev_mc_block_seqno{0};
ton::BlockSeqno new_block_seqno{0};
@ -92,7 +94,7 @@ class Collator final : public td::actor::Actor {
Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev,
Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id, Ref<CollatorOptions> collator_opts,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout, td::Promise<BlockCandidate> promise,
td::CancellationToken cancellation_token, unsigned mode);
td::CancellationToken cancellation_token, unsigned mode, int attempt_idx);
~Collator() override = default;
bool is_busy() const {
return busy_;
@ -330,6 +332,7 @@ class Collator final : public td::actor::Actor {
bool insert_out_msg(Ref<vm::Cell> out_msg);
bool insert_out_msg(Ref<vm::Cell> out_msg, td::ConstBitPtr msg_hash);
bool register_out_msg_queue_op(bool force = false);
bool register_dispatch_queue_op(bool force = false);
bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno);
bool combine_account_transactions();
bool update_public_libraries();

96
validator/impl/collator.cpp
View file

@ -45,11 +45,13 @@ using td::Ref;
using namespace std::literals::string_literals;
// Don't increase MERGE_MAX_QUEUE_LIMIT too much: merging requires cleaning the whole queue in out_msg_queue_cleanup
static const td::uint32 FORCE_SPLIT_QUEUE_SIZE = 4096;
static const td::uint32 SPLIT_MAX_QUEUE_SIZE = 100000;
static const td::uint32 MERGE_MAX_QUEUE_SIZE = 2047;
static const td::uint32 SKIP_EXTERNALS_QUEUE_SIZE = 8000;
static const int HIGH_PRIORITY_EXTERNAL = 10; // don't skip high priority externals when queue is big
static constexpr td::uint32 FORCE_SPLIT_QUEUE_SIZE = 4096;
static constexpr td::uint32 SPLIT_MAX_QUEUE_SIZE = 100000;
static constexpr td::uint32 MERGE_MAX_QUEUE_SIZE = 2047;
static constexpr td::uint32 SKIP_EXTERNALS_QUEUE_SIZE = 8000;
static constexpr int HIGH_PRIORITY_EXTERNAL = 10; // don't skip high priority externals when queue is big
static constexpr int MAX_ATTEMPTS = 5;
#define DBG(__n) dbg(__n)&&
#define DSTART int __dcnt = 0;
@ -74,12 +76,15 @@ static inline bool dbg(int c) {
* @param manager The ActorId of the ValidatorManager.
* @param timeout The timeout for the collator.
* @param promise The promise to return the result.
* @param cancellation_token Token to cancel collation.
* @param mode +1 - skip storing candidate to disk.
* @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps.
*/
Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id,
std::vector<BlockIdExt> prev, td::Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id,
Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager,
td::Timestamp timeout, td::Promise<BlockCandidate> promise, td::CancellationToken cancellation_token,
unsigned mode)
unsigned mode, int attempt_idx)
: shard_(shard)
, is_hardfork_(is_hardfork)
, min_mc_block_id{min_masterchain_block_id}
@ -95,6 +100,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha
, medium_timeout_(td::Timestamp::at(timeout.at() - 1.5))
, main_promise(std::move(promise))
, mode_(mode)
, attempt_idx_(attempt_idx)
, perf_timer_("collate", 0.1,
[manager](double duration) {
send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration);
@ -111,7 +117,11 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha
* The results of these queries are handled by corresponding callback functions.
*/
void Collator::start_up() {
LOG(WARNING) << "Collator for shard " << shard_.to_str() << " started";
LOG(WARNING) << "Collator for shard " << shard_.to_str() << " started"
<< (attempt_idx_ ? PSTRING() << " (attempt #" << attempt_idx_ << ")" : "");
if (!check_cancelled()) {
return;
}
LOG(DEBUG) << "Previous block #1 is " << prev_blocks.at(0).to_str();
if (prev_blocks.size() > 1) {
LOG(DEBUG) << "Previous block #2 is " << prev_blocks.at(1).to_str();
@ -344,8 +354,15 @@ bool Collator::fatal_error(td::Status error) {
error.ensure_error();
LOG(ERROR) << "cannot generate block candidate for " << show_shard(shard_) << " : " << error.to_string();
if (busy_) {
LOG(INFO) << "collation took " << perf_timer_.elapsed() << " s";
main_promise(std::move(error));
if (allow_repeat_collation_ && error.code() != ErrorCode::cancelled && attempt_idx_ + 1 < MAX_ATTEMPTS &&
!is_hardfork_ && !timeout.is_in_past()) {
LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")";
run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager,
td::Timestamp::in(10.0), std::move(main_promise), std::move(cancellation_token_), mode_,
attempt_idx_ + 1);
} else {
main_promise(std::move(error));
}
busy_ = false;
}
stop();
@ -728,6 +745,15 @@ bool Collator::unpack_last_mc_state() {
return fatal_error(limits.move_as_error());
}
block_limits_ = limits.move_as_ok();
if (attempt_idx_ == 3) {
LOG(INFO) << "Attempt #3: bytes, gas limits /= 2";
block_limits_->bytes.multiply_by(0.5);
block_limits_->gas.multiply_by(0.5);
} else if (attempt_idx_ == 4) {
LOG(INFO) << "Attempt #4: bytes, gas limits /= 4";
block_limits_->bytes.multiply_by(0.25);
block_limits_->gas.multiply_by(0.25);
}
LOG(DEBUG) << "block limits: bytes [" << block_limits_->bytes.underload() << ", " << block_limits_->bytes.soft()
<< ", " << block_limits_->bytes.hard() << "]";
LOG(DEBUG) << "block limits: gas [" << block_limits_->gas.underload() << ", " << block_limits_->gas.soft() << ", "
@ -2189,6 +2215,7 @@ bool Collator::do_collate() {
if (max_lt == start_lt) {
++max_lt;
}
allow_repeat_collation_ = true;
// NB: interchanged 1.2 and 1.1 (is this always correct?)
// 1.1. re-adjust neighbors' out_msg_queues (for oneself)
if (!add_trivial_neighbor()) {
@ -3715,6 +3742,10 @@ bool Collator::process_inbound_external_messages() {
LOG(INFO) << "skipping processing of inbound external messages";
return true;
}
if (attempt_idx_ >= 2) {
LOG(INFO) << "Attempt #" << attempt_idx_ << ": skip external messages";
return true;
}
if (out_msg_queue_size_ > SKIP_EXTERNALS_QUEUE_SIZE) {
LOG(INFO) << "skipping processing of inbound external messages (except for high-priority) because out_msg_queue is "
"too big ("
@ -3845,6 +3876,10 @@ bool Collator::process_dispatch_queue() {
if (max_per_initiator[iter] == 0 || max_total_count[iter] == 0) {
continue;
}
if (iter > 0 && attempt_idx_ >= 1) {
LOG(INFO) << "Attempt #" << attempt_idx_ << ": skip process_dispatch_queue";
break;
}
vm::AugmentedDictionary cur_dispatch_queue{dispatch_queue_->get_root(), 256, block::tlb::aug_DispatchQueue};
std::map<std::tuple<WorkchainId, StdSmcAddress, LogicalTime>, size_t> count_per_initiator;
size_t total_count = 0;
@ -3857,13 +3892,13 @@ bool Collator::process_dispatch_queue() {
stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": "
<< block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal)
<< "\n";
return true;
return register_dispatch_queue_op(true);
}
if (soft_timeout_.is_in_past(td::Timestamp::now())) {
block_full_ = true;
LOG(WARNING) << "soft timeout reached, stop processing dispatch queue";
stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": timeout\n";
return true;
return register_dispatch_queue_op(true);
}
StdSmcAddress src_addr;
td::Ref<vm::CellSlice> account_dispatch_queue;
@ -3941,6 +3976,7 @@ bool Collator::process_dispatch_queue() {
if (iter == 0) {
have_unprocessed_account_dispatch_queue_ = false;
}
register_dispatch_queue_op(true);
}
return true;
}
@ -3964,12 +4000,7 @@ bool Collator::process_deferred_message(Ref<vm::CellSlice> enq_msg, StdSmcAddres
return fatal_error(PSTRING() << "failed to delete message from DispatchQueue: address=" << src_addr.to_hex()
<< ", lt=" << lt);
}
++dispatch_queue_ops_;
if (!(dispatch_queue_ops_ & 63)) {
if (!block_limit_status_->add_proof(dispatch_queue_->get_root_cell())) {
return false;
}
}
register_dispatch_queue_op();
++sender_generated_messages_count_[src_addr];
LogicalTime enqueued_lt = 0;
@ -4062,6 +4093,7 @@ bool Collator::process_deferred_message(Ref<vm::CellSlice> enq_msg, StdSmcAddres
++unprocessed_deferred_messages_[src_addr];
LOG(INFO) << "delivering deferred message from account " << src_addr.to_hex() << ", lt=" << lt
<< ", emitted_lt=" << emitted_lt;
block_limit_status_->add_cell(msg_env);
register_new_msg(std::move(new_msg));
msg_metadata = std::move(env.metadata);
return true;
@ -4241,11 +4273,7 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema
}
++dispatch_dict_size;
dispatch_queue_->set(src_addr, block::pack_account_dispatch_queue(dispatch_dict, dispatch_dict_size));
++dispatch_queue_ops_;
if (!(dispatch_queue_ops_ & 63)) {
return block_limit_status_->add_proof(dispatch_queue_->get_root_cell());
}
return true;
return register_dispatch_queue_op();
}
auto next_hop = block::interpolate_addr(src_prefix, dest_prefix, route_info.second);
@ -5130,6 +5158,23 @@ bool Collator::register_out_msg_queue_op(bool force) {
}
}
/**
* Registers a dispatch queue message queue operation.
* Adds the proof to the block limit status every 64 operations.
*
* @param force If true, the proof will always be added to the block limit status.
*
* @returns True if the operation was successfully registered, false otherwise.
*/
bool Collator::register_dispatch_queue_op(bool force) {
++dispatch_queue_ops_;
if (force || !(dispatch_queue_ops_ & 63)) {
return block_limit_status_->add_proof(dispatch_queue_->get_root_cell());
} else {
return true;
}
}
/**
* Creates a new shard state and the Merkle update.
*
@ -5255,9 +5300,10 @@ bool Collator::compute_out_msg_queue_info(Ref<vm::Cell>& out_msg_queue_info) {
vm::CellSlice maybe_extra = cb.as_cellslice();
cb.reset();
return register_out_msg_queue_op(true) && out_msg_queue_->append_dict_to_bool(cb) // _ out_queue:OutMsgQueue
&& processed_upto_->pack(cb) // proc_info:ProcessedInfo
&& cb.append_cellslice_bool(maybe_extra) // extra:(Maybe OutMsgQueueExtra)
return register_out_msg_queue_op(true) && register_dispatch_queue_op(true) &&
out_msg_queue_->append_dict_to_bool(cb) // _ out_queue:OutMsgQueue
&& processed_upto_->pack(cb) // proc_info:ProcessedInfo
&& cb.append_cellslice_bool(maybe_extra) // extra:(Maybe OutMsgQueueExtra)
&& cb.finalize_to(out_msg_queue_info);
}

13
validator/impl/fabric.cpp
View file

@ -216,17 +216,18 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc
Ed25519_PublicKey creator, td::Ref<ValidatorSet> validator_set,
td::Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager,
td::Timestamp timeout, td::Promise<BlockCandidate> promise,
td::CancellationToken cancellation_token, unsigned mode) {
td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) {
BlockSeqno seqno = 0;
for (auto& p : prev) {
if (p.seqno() > seqno) {
seqno = p.seqno();
}
}
td::actor::create_actor<Collator>(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false,
min_masterchain_block_id, std::move(prev), std::move(validator_set), creator,
std::move(collator_opts), std::move(manager), timeout, std::move(promise),
std::move(cancellation_token), mode)
td::actor::create_actor<Collator>(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1)
<< (attempt_idx ? "_" + td::to_string(attempt_idx) : ""),
shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set),
creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise),
std::move(cancellation_token), mode, attempt_idx)
.release();
}
@ -242,7 +243,7 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b
td::actor::create_actor<Collator>(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true,
min_masterchain_block_id, std::move(prev), td::Ref<ValidatorSet>{},
Ed25519_PublicKey{Bits256::zero()}, td::Ref<CollatorOptions>{true},
std::move(manager), timeout, std::move(promise), td::CancellationToken{}, 0)
std::move(manager), timeout, std::move(promise), td::CancellationToken{}, 0, 0)
.release();
}

10
validator/impl/liteserver.cpp
View file

@ -1377,13 +1377,9 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) {
if (acc_root.not_null()) {
if (mode_ & 0x40000000) {
vm::MerkleProofBuilder mpb{acc_root};
// account_none$0 = Account;
// account$1 addr:MsgAddressInt storage_stat:StorageInfo storage:AccountStorage = Account;
// account_storage$_ last_trans_lt:uint64 balance:CurrencyCollection state:AccountState = AccountStorage;
// account_active$1 _:StateInit = AccountState;
auto S = mpb.root()->load_cell();
if (S.is_error()) {
fatal_error(S.move_as_error_prefix("Failed to load account: "));
// This does not include code, data and libs into proof, but it includes extra currencies
if (!block::gen::t_Account.validate_ref(mpb.root())) {
fatal_error("failed to validate Account");
return;
}
if (!mpb.extract_proof_to(acc_root)) {

2
validator/manager-disk.cpp
View file

@ -129,7 +129,7 @@ void ValidatorManagerImpl::sync_complete(td::Promise<td::Unit> promise) {
Ed25519_PublicKey created_by{td::Bits256::zero()};
td::as<td::uint32>(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8);
run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref<CollatorOptions>{true},
actor_id(this), td::Timestamp::in(10.0), std::move(P));
actor_id(this), td::Timestamp::in(10.0), std::move(P), td::CancellationToken{}, 0);
}
void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector<BlockIdExt> prev, BlockIdExt last,

5
validator/validator-group.cpp
View file

@ -172,6 +172,7 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s
prev_block_ids_ = std::vector<BlockIdExt>{next_block_id};
cached_collated_block_ = nullptr;
approved_candidates_cache_.clear();
cancellation_token_source_.cancel();
}
void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref<BlockData> block, std::vector<BlockIdExt> prev,
@ -417,6 +418,7 @@ void ValidatorGroup::destroy() {
delay_action([ses]() mutable { td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); },
td::Timestamp::in(10.0));
}
cancellation_token_source_.cancel();
stop();
}
@ -506,7 +508,8 @@ void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, t
if (self_collate) {
run_collate_query(shard_, min_masterchain_block_id_, prev_block_ids_,
Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_,
opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise));
opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise),
cancellation_token_source_.get_cancellation_token(), 0);
return;
}
if (collator_adnl_id.is_zero()) {

1
validator/validator-group.hpp
View file

@ -148,6 +148,7 @@ class ValidatorGroup : public td::actor::Actor {
std::vector<td::Promise<BlockCandidate>> promises;
};
std::shared_ptr<CachedCollatedBlock> cached_collated_block_;
td::CancellationTokenSource cancellation_token_source_;
void generated_block_candidate(std::shared_ptr<CachedCollatedBlock> cache, td::Result<BlockCandidate> R);