external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Lite-mode for validate-query, more collated data in collator

Browse source
This commit is contained in:
SpyCheese 2022-07-12 11:04:31 +03:00
parent 849d995346
commit 625516c568
7 changed files with 214 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

4
validator/fabric.h
View file

@ -25,6 +25,8 @@ namespace ton {
namespace validator {
enum ValidateMode { fake = 1, lite = 2 };
td::actor::ActorOwn<Db> create_db_actor(td::actor::ActorId<ValidatorManager> manager, std::string db_root_);
td::actor::ActorOwn<LiteServerCache> create_liteserver_cache_actor(td::actor::ActorId<ValidatorManager> manager,
std::string db_root);
@ -73,7 +75,7 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref<ProofLink> proof, td::act
void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id,
std::vector<BlockIdExt> prev, BlockCandidate candidate, td::Ref<ValidatorSet> validator_set,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
td::Promise<ValidateCandidateResult> promise, bool is_fake = false);
td::Promise<ValidateCandidateResult> promise, unsigned mode = 0);
void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id,
std::vector<BlockIdExt> prev, Ed25519_PublicKey local_id, td::Ref<ValidatorSet> validator_set,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,

7
validator/impl/collator-impl.h
View file

@ -205,7 +205,11 @@ class Collator final : public td::actor::Actor {
unsigned block_create_total_{0};
std::vector<ExtMessage::Hash> bad_ext_msgs_, delay_ext_msgs_;
Ref<vm::Cell> shard_account_blocks_; // ShardAccountBlocks
std::map<td::Bits256, Ref<BlockData>> blocks_with_state_proofs_;
std::vector<vm::MerkleProofBuilder> neighbor_proof_builders_;
std::vector<Ref<vm::Cell>> collated_roots_;
std::unique_ptr<ton::BlockCandidate> block_candidate;
td::PerfWarningTimer perf_timer_{"collate", 0.1};
@ -235,7 +239,8 @@ class Collator final : public td::actor::Actor {
void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result<Ref<ShardState>> res);
bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner);
bool fix_processed_upto(block::MsgProcessedUptoCollection& upto);
void got_neighbor_out_queue(int i, td::Result<Ref<MessageQueue>> res);
void got_neighbor_block_data(td::Result<Ref<BlockData>> res);
void got_neighbor_block_state(int i, td::Result<Ref<ShardState>> res);
bool adjust_shard_config();
bool store_shard_fees(ShardIdFull shard, const block::CurrencyCollection& fees,
const block::CurrencyCollection& created);

89
validator/impl/collator.cpp
View file

@ -502,6 +502,7 @@ void Collator::after_get_block_data(int idx, td::Result<Ref<BlockData>> res) {
prev_mc_block = prev_block_data[0];
mc_block_root = prev_mc_block->root_cell();
}
blocks_with_state_proofs_[prev_block_data[idx]->root_cell()->get_hash().bits()] = prev_block_data[idx];
}
check_pending();
}
@ -614,26 +615,59 @@ bool Collator::request_neighbor_msg_queues() {
neighbors_.emplace_back(*shard_ptr);
}
int i = 0;
neighbor_proof_builders_.resize(neighbors_.size());
for (block::McShardDescr& descr : neighbors_) {
LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str();
if (descr.blk_.seqno() != 0) {
++pending;
send_closure_later(manager, &ValidatorManager::wait_block_data_short, descr.blk_, priority(), timeout,
[self = get_self(), i](td::Result<Ref<BlockData>> res) {
LOG(DEBUG) << "got answer to wait_block_data for neighbor #" << i;
send_closure_later(std::move(self), &Collator::got_neighbor_block_data, std::move(res));
});
}
++pending;
send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout,
[self = get_self(), i](td::Result<Ref<MessageQueue>> res) {
td::actor::send_closure(std::move(self), &Collator::got_neighbor_out_queue, i, std::move(res));
send_closure_later(manager, &ValidatorManager::wait_block_state_short, descr.blk_, priority(), timeout,
[self = get_self(), i](td::Result<Ref<ShardState>> res) {
LOG(DEBUG) << "got answer to wait_block_state for neighbor #" << i;
send_closure_later(std::move(self), &Collator::got_neighbor_block_state, i, std::move(res));
});
++i;
}
return true;
}
void Collator::got_neighbor_out_queue(int i, td::Result<Ref<MessageQueue>> res) {
LOG(DEBUG) << "obtained outbound queue for neighbor #" << i;
void Collator::got_neighbor_block_data(td::Result<Ref<BlockData>> res) {
--pending;
if (res.is_error()) {
fatal_error(res.move_as_error());
return;
}
Ref<MessageQueue> outq_descr = res.move_as_ok();
auto block_data = res.move_as_ok();
blocks_with_state_proofs_[block_data->root_cell()->get_hash().bits()] = block_data;
check_pending();
}
void Collator::got_neighbor_block_state(int i, td::Result<Ref<ShardState>> res) {
--pending;
if (res.is_error()) {
fatal_error(res.move_as_error());
return;
}
Ref<ShardState> state = res.move_as_ok();
neighbor_proof_builders_.at(i) = vm::MerkleProofBuilder{state->root_cell()};
auto new_state = ShardStateQ::fetch(state->get_block_id(), {}, neighbor_proof_builders_.at(i).root());
if (new_state.is_error()) {
fatal_error(new_state.move_as_error());
return;
}
auto outq_descr_res = new_state.move_as_ok()->message_queue();
if (outq_descr_res.is_error()) {
fatal_error(outq_descr_res.move_as_error());
return;
}
LOG(DEBUG) << "obtained outbound queue for neighbor #" << i;
Ref<MessageQueue> outq_descr = outq_descr_res.move_as_ok();
block::McShardDescr& descr = neighbors_.at(i);
if (outq_descr->get_block_id() != descr.blk_) {
LOG(DEBUG) << "outq_descr->id = " << outq_descr->get_block_id().to_str() << " ; descr.id = " << descr.blk_.to_str();
@ -3949,7 +3983,6 @@ Ref<vm::Cell> Collator::collate_shard_block_descr_set() {
}
bool Collator::create_collated_data() {
// TODO: store something into collated_roots_
// 1. store the set of used shard block descriptions
if (!used_shard_block_descr_.empty()) {
auto cell = collate_shard_block_descr_set();
@ -3959,7 +3992,47 @@ bool Collator::create_collated_data() {
}
collated_roots_.push_back(std::move(cell));
}
// 2. ...
// 2. Proofs for hashes of states: previous states + neighbors
for (const auto& p : blocks_with_state_proofs_) {
vm::MerkleProofBuilder mpb{p.second->root_cell()};
block::gen::Block::Record block;
if (!tlb::unpack_cell(mpb.root(), block) || block.state_update->load_cell().is_error()) {
return fatal_error("cannot generate Merkle proof for previous block");
}
Ref<vm::Cell> proof = mpb.extract_proof();
if (proof.is_null()) {
return fatal_error("cannot generate Merkle proof for previous block");
}
collated_roots_.push_back(std::move(proof));
}
// 3. Previous state proof (only shadchains)
std::map<td::Bits256, Ref<vm::Cell>> proofs;
if (!is_masterchain()) {
state_usage_tree_->set_use_mark_for_is_loaded(false);
Ref<vm::Cell> state_proof = vm::MerkleProof::generate(prev_state_root_, state_usage_tree_.get());
if (state_proof.is_null()) {
return fatal_error("cannot generate Merkle proof for previous state");
}
proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof);
}
// 4. Proofs for message queues
for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) {
Ref<vm::Cell> proof = mpb.extract_proof();
if (proof.is_null()) {
return fatal_error("cannot generate Merkle proof for neighbor");
}
auto it = proofs.emplace(mpb.root()->get_hash().bits(), proof);
if (!it.second) {
it.first->second = vm::MerkleProof::combine(it.first->second, std::move(proof));
if (it.first->second.is_null()) {
return fatal_error("cannot combine merkle proofs");
}
}
}
for (auto& p : proofs) {
collated_roots_.push_back(std::move(p.second));
}
return true;
}

5
validator/impl/fabric.cpp
View file

@ -191,17 +191,18 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref<ProofLink> proof, td::act
void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id,
std::vector<BlockIdExt> prev, BlockCandidate candidate, td::Ref<ValidatorSet> validator_set,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
td::Promise<ValidateCandidateResult> promise, bool is_fake) {
td::Promise<ValidateCandidateResult> promise, unsigned mode) {
BlockSeqno seqno = 0;
for (auto& p : prev) {
if (p.seqno() > seqno) {
seqno = p.seqno();
}
}
bool is_fake = mode & ValidateMode::fake;
td::actor::create_actor<ValidateQuery>(
PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() << ":" << (seqno + 1), shard, min_ts,
min_masterchain_block_id, std::move(prev), std::move(candidate), std::move(validator_set), std::move(manager),
timeout, std::move(promise), is_fake)
timeout, std::move(promise), mode)
.release();
}

138
validator/impl/validate-query.cpp
View file

@ -31,6 +31,7 @@
#include "vm/cells/MerkleProof.h"
#include "vm/cells/MerkleUpdate.h"
#include "common/errorlog.h"
#include "fabric.h"
#include <ctime>
namespace ton {
@ -51,7 +52,7 @@ std::string ErrorCtx::as_string() const {
ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id,
std::vector<BlockIdExt> prev, BlockCandidate candidate, Ref<ValidatorSet> validator_set,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
td::Promise<ValidateCandidateResult> promise, bool is_fake)
td::Promise<ValidateCandidateResult> promise, unsigned mode)
: shard_(shard)
, id_(candidate.id)
, min_ts(min_ts)
@ -62,7 +63,8 @@ ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_
, manager(std::move(manager))
, timeout(timeout)
, main_promise(std::move(promise))
, is_fake_(is_fake)
, is_fake_(mode & ValidateMode::fake)
, is_lite_(mode & ValidateMode::lite)
, shard_pfx_(shard_.shard)
, shard_pfx_len_(ton::shard_prefix_length(shard_)) {
proc_hash_.zero();
@ -257,18 +259,20 @@ void ValidateQuery::start_up() {
td::actor::send_closure_later(
std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res));
});
// 3. load state(s) corresponding to previous block(s)
// 3. load state(s) corresponding to previous block(s) (non-lite mode or masterchain)
prev_states.resize(prev_blocks.size());
for (int i = 0; (unsigned)i < prev_blocks.size(); i++) {
// 3.1. load state
LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager";
++pending;
td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(),
timeout, [self = get_self(), i](td::Result<Ref<ShardState>> res) -> void {
LOG(DEBUG) << "got answer to wait_block_state_short query #" << i;
td::actor::send_closure_later(
std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res));
});
if (is_masterchain() || !is_lite_) {
for (int i = 0; (unsigned)i < prev_blocks.size(); i++) {
// 3.1. load state
LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager";
++pending;
td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(),
timeout, [self = get_self(), i](td::Result<Ref<ShardState>> res) -> void {
LOG(DEBUG) << "got answer to wait_block_state_short query #" << i;
td::actor::send_closure_later(
std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res));
});
}
}
// 4. unpack block candidate (while necessary data is being loaded)
if (!unpack_block_candidate()) {
@ -988,6 +992,9 @@ bool ValidateQuery::check_this_shard_mc_info() {
*/
bool ValidateQuery::compute_prev_state() {
if (!is_masterchain() && is_lite_) {
return compute_prev_state_lite_mode();
}
CHECK(prev_states.size() == 1u + after_merge_);
prev_state_root_ = prev_states[0]->root_cell();
CHECK(prev_state_root_.not_null());
@ -1006,6 +1013,46 @@ bool ValidateQuery::compute_prev_state() {
return true;
}
bool ValidateQuery::compute_prev_state_lite_mode() {
td::Bits256 state_hash;
if (id_.seqno() == 1) {
if (prev_blocks.size() != 1) {
return reject_query("seqno is 1, but number of previous blocks is not 1");
}
state_hash = prev_blocks[0].root_hash;
} else {
std::vector<Ref<vm::Cell>> prev_state_roots(prev_blocks.size());
for (size_t i = 0; i < prev_blocks.size(); ++i) {
prev_state_roots[i] = get_virt_state_root(prev_blocks[i].root_hash);
if (prev_state_roots[i].is_null()) {
return reject_query(PSTRING() << "cannot get hash of previous state root: " << prev_blocks[i]);
}
}
if (prev_state_roots.size() == 1) {
state_hash = prev_state_roots[0]->get_hash().bits();
} else {
CHECK(prev_state_roots.size() == 2);
Ref<vm::Cell> merged;
if (!block::gen::t_ShardState.cell_pack_split_state(merged, prev_state_roots[0], prev_state_roots[1])) {
return fatal_error(-667, "cannot construct mechanically merged previously state");
}
state_hash = merged->get_hash().bits();
}
}
if (state_hash != prev_state_hash_) {
return reject_query("previous state hash mismatch for block "s + id_.to_str() + " : block header declares " +
prev_state_hash_.to_hex() + " , actual " + state_hash.to_hex());
}
auto it = virt_roots_.find(state_hash);
if (it == virt_roots_.end()) {
return reject_query(PSTRING() << "no state root for previous block in collated data (hash = "
<< state_hash.to_hex() << ")");
}
prev_state_root_ = it->second;
return true;
}
bool ValidateQuery::compute_next_state() {
LOG(DEBUG) << "computing next state";
auto res = vm::MerkleUpdate::validate(state_update_);
@ -1209,15 +1256,42 @@ bool ValidateQuery::request_neighbor_queues() {
neighbors_.emplace_back(*shard_ptr);
}
int i = 0;
for (block::McShardDescr& descr : neighbors_) {
LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str();
++pending;
send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout,
[self = get_self(), i](td::Result<Ref<MessageQueue>> res) {
td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i,
std::move(res));
});
++i;
if (is_lite_) {
for (block::McShardDescr& descr : neighbors_) {
LOG(DEBUG) << "getting outbound queue of neighbor #" << i << " from collated data : " << descr.blk_.to_str();
td::Bits256 state_root_hash;
if (descr.blk_.seqno() == 0) {
state_root_hash = descr.blk_.root_hash;
} else {
Ref<vm::Cell> state_root = get_virt_state_root(descr.blk_.root_hash);
if (state_root.is_null()) {
return reject_query(PSTRING() << "cannot get hash of state root: " << descr.blk_);
}
state_root_hash = state_root->get_hash().bits();
}
auto it = virt_roots_.find(state_root_hash);
if (it == virt_roots_.end()) {
return reject_query(PSTRING() << "cannot get state root form collated data: " << descr.blk_);
}
auto state = ShardStateQ::fetch(descr.blk_, {}, it->second);
if (state.is_error()) {
return reject_query("cannot fetch shard state from collated data", state.move_as_error());
}
++pending;
send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, state.move_as_ok()->message_queue());
++i;
}
} else {
for (block::McShardDescr& descr : neighbors_) {
LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str();
++pending;
send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout,
[self = get_self(), i](td::Result<Ref<MessageQueue>> res) {
td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i,
std::move(res));
});
++i;
}
}
return true;
}
@ -5430,6 +5504,24 @@ bool ValidateQuery::check_mc_block_extra() {
return true;
}
Ref<vm::Cell> ValidateQuery::get_virt_state_root(td::Bits256 block_root_hash) {
auto it = virt_roots_.find(block_root_hash);
if (it == virt_roots_.end()) {
return {};
}
Ref<vm::Cell> root = it->second;
block::gen::Block::Record block;
if (!tlb::unpack_cell(root, block)) {
return {};
}
vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update};
if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 // merkle update
&& upd_cs.size_ext() == 0x20228)) {
return {};
}
return vm::MerkleProof::virtualize_raw(upd_cs.prefetch_ref(1), {0, 1});
}
/*
*
* MAIN VALIDATOR FUNCTION
@ -5533,7 +5625,7 @@ bool ValidateQuery::try_validate() {
} catch (vm::VmError& err) {
return fatal_error(-666, err.get_msg());
} catch (vm::VmVirtError& err) {
return fatal_error(-666, err.get_msg());
return reject_query(err.get_msg());
}
return save_candidate();
}

6
validator/impl/validate-query.hpp
View file

@ -118,7 +118,7 @@ class ValidateQuery : public td::actor::Actor {
ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector<BlockIdExt> prev,
BlockCandidate candidate, td::Ref<ValidatorSet> validator_set,
td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
td::Promise<ValidateCandidateResult> promise, bool is_fake = false);
td::Promise<ValidateCandidateResult> promise, unsigned mode = 0);
private:
int verbosity{3 * 1};
@ -142,6 +142,7 @@ class ValidateQuery : public td::actor::Actor {
bool is_key_block_{false};
bool update_shard_cc_{false};
bool is_fake_{false};
bool is_lite_{false};
bool prev_key_block_exists_{false};
bool debug_checks_{false};
bool outq_cleanup_partial_{false};
@ -286,6 +287,7 @@ class ValidateQuery : public td::actor::Actor {
bool extract_collated_data();
bool try_validate();
bool compute_prev_state();
bool compute_prev_state_lite_mode();
bool compute_next_state();
bool unpack_merge_prev_state();
bool unpack_prev_state();
@ -368,6 +370,8 @@ class ValidateQuery : public td::actor::Actor {
bool check_one_shard_fee(ShardIdFull shard, const block::CurrencyCollection& fees,
const block::CurrencyCollection& create);
bool check_mc_block_extra();
Ref<vm::Cell> get_virt_state_root(td::Bits256 block_root_hash);
};
} // namespace validator

2
validator/manager-disk.cpp
View file

@ -153,7 +153,7 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector<B
});
auto shard = candidate.id.shard_full();
run_validate_query(shard, 0, last, prev, std::move(candidate), std::move(val_set), actor_id(this),
td::Timestamp::in(10.0), std::move(P), true /* fake */);
td::Timestamp::in(10.0), std::move(P), ValidateMode::fake);
}
void ValidatorManagerImpl::write_fake(BlockCandidate candidate, std::vector<BlockIdExt> prev, BlockIdExt last,