1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00

Tonlib improvement fixes (#934)

* 3.6. Fix directory traversal in KeyValueDir

* 3.9. Fix buffer_to_hex reversing nibbles

* 3.5. Fix error handling at blocks.getBlockHeader

* 3.11. Fix query.forget

* 3.12. Fix error handling in RemoteRunSmcMethod

* 4.1. Delete unused files

* 3.10. Use named constants instead hardcoded constants

* 3.4. Fix response block header verification

* 3.1. Check proof of blocks.getShards response

* fix td::buffer_to_hex + test

* 3.2. Add proof check for listBlockTransactions response in RunEmulator actor

* 3.8. Add proof checking for getLibraries method

* fix regression tests

* 3.3 Add proof checking for lookupBlock method

* Add publishers to proof of getLibrariesWithProof response  (#25)

* fix missing return, fix requesting mc block

* Fix requesting lookupBlock with client mc blk == mc ref block

* Fix duplicating lib data in proof and data, add mode 2 for not including the data

* Migration of LastBlockStorage with fixed td::buffer_to_hex

---------

Co-authored-by: ms <dungeon666master@protonmail.com>
Co-authored-by: Marat <98183742+dungeon-master-666@users.noreply.github.com>
This commit is contained in:
EmelyanenkoK 2024-03-19 15:31:29 +03:00 committed by GitHub
parent 69de1cb621
commit f1592641de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 1000 additions and 854 deletions

View file

@ -233,6 +233,9 @@ void LiteQuery::perform() {
[&](lite_api::liteServer_lookupBlock& q) {
this->perform_lookupBlock(ton::create_block_id_simple(q.id_), q.mode_, q.lt_, q.utime_);
},
[&](lite_api::liteServer_lookupBlockWithProof& q) {
this->perform_lookupBlockWithProof(ton::create_block_id_simple(q.id_), ton::create_block_id(q.mc_block_id_), q.mode_, q.lt_, q.utime_);
},
[&](lite_api::liteServer_listBlockTransactions& q) {
this->perform_listBlockTransactions(ton::create_block_id(q.id_), q.mode_, q.count_,
(q.mode_ & 128) ? q.after_->account_ : td::Bits256::zero(),
@ -266,6 +269,9 @@ void LiteQuery::perform() {
[&](lite_api::liteServer_getLibraries& q) {
this->perform_getLibraries(q.library_list_);
},
[&](lite_api::liteServer_getLibrariesWithProof& q) {
this->perform_getLibrariesWithProof(ton::create_block_id(q.id_), q.mode_, q.library_list_);
},
[&](lite_api::liteServer_getShardBlockProof& q) {
this->perform_getShardBlockProof(create_block_id(q.id_));
},
@ -964,6 +970,100 @@ void LiteQuery::continue_getLibraries(Ref<ton::validator::MasterchainState> mc_s
finish_query(std::move(b));
}
void LiteQuery::perform_getLibrariesWithProof(BlockIdExt blkid, int mode, std::vector<td::Bits256> library_list) {
LOG(INFO) << "started a getLibrariesWithProof(<list of " << library_list.size() << " parameters>) liteserver query";
if (library_list.size() > 16) {
LOG(INFO) << "too many libraries requested, returning only first 16";
library_list.resize(16);
}
sort( library_list.begin(), library_list.end() );
library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() );
set_continuation([this, library_list, mode]() -> void { continue_getLibrariesWithProof(library_list, mode); });
request_mc_block_data_state(blkid);
}
void LiteQuery::continue_getLibrariesWithProof(std::vector<td::Bits256> library_list, int mode) {
LOG(INFO) << "obtained masterchain block = " << base_blk_id_.to_str();
CHECK(mc_state_.not_null());
Ref<vm::Cell> state_proof, data_proof;
if (!make_mc_state_root_proof(state_proof)) {
return;
}
vm::MerkleProofBuilder pb{mc_state_->root_cell()};
block::gen::ShardStateUnsplit::Record state;
if (!tlb::unpack_cell(pb.root(), state)) {
fatal_error("cannot unpack header of shardchain state "s + base_blk_id_.to_str());
}
auto libraries_dict = vm::Dictionary(state.r1.libraries->prefetch_ref(), 256);
std::vector<ton::tl_object_ptr<ton::lite_api::liteServer_libraryEntry>> result;
std::vector<td::Bits256> result_hashes;
for (const auto& hash : library_list) {
LOG(INFO) << "looking for library " << hash.to_hex();
auto csr = libraries_dict.lookup(hash.bits(), 256);
if (csr.is_null() || csr->prefetch_ulong(2) != 0 || !csr->have_refs()) { // shared_lib_descr$00 lib:^Cell
continue;
}
block::gen::LibDescr::Record libdescr;
if (!tlb::csr_unpack(csr, libdescr)) {
fatal_error("cannot unpack LibDescr record "s + hash.to_hex());
return;
}
if (mode & 1) {
// include first 16 publishers in the proof
auto publishers_dict = vm::Dictionary{vm::DictNonEmpty(), libdescr.publishers, 256};
auto iter = publishers_dict.begin();
constexpr int max_publishers = 15; // set to 15 because publishers_dict.begin() counts as the first visit
for (int i = 0; i < max_publishers && iter != publishers_dict.end(); ++i, ++iter) {}
}
result_hashes.push_back(hash);
}
auto data_proof_boc = pb.extract_proof_boc();
if (data_proof_boc.is_error()) {
fatal_error(data_proof_boc.move_as_error());
return;
}
auto state_proof_boc = vm::std_boc_serialize(std::move(state_proof));
if (state_proof_boc.is_error()) {
fatal_error(state_proof_boc.move_as_error());
return;
}
for (const auto& hash : result_hashes) {
auto csr = libraries_dict.lookup(hash.bits(), 256);
block::gen::LibDescr::Record libdescr;
if (!tlb::csr_unpack(csr, libdescr)) {
fatal_error("cannot unpack LibDescr record "s + hash.to_hex());
return;
}
if (!libdescr.lib->get_hash().bits().equals(hash.bits(), 256)) {
LOG(ERROR) << "public library hash mismatch: expected " << hash.to_hex() << " , found "
<< libdescr.lib->get_hash().to_hex();
continue;
}
td::BufferSlice libdata;
if (!(mode & 2)) {
auto data = vm::std_boc_serialize(libdescr.lib);
if (data.is_error()) {
LOG(WARNING) << "library serialization failed: " << data.move_as_error().to_string();
continue;
}
libdata = data.move_as_ok();
}
result.push_back(ton::create_tl_object<ton::lite_api::liteServer_libraryEntry>(hash, std::move(libdata)));
}
auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_libraryResultWithProof>(ton::create_tl_lite_block_id(base_blk_id_), mode, std::move(result),
state_proof_boc.move_as_ok(), data_proof_boc.move_as_ok());
finish_query(std::move(b));
}
void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt) {
LOG(INFO) << "started a getOneTransaction(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ","
<< lt << ") liteserver query";
@ -1853,7 +1953,7 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector<int> param_list)
void LiteQuery::perform_getAllShardsInfo(BlockIdExt blkid) {
LOG(INFO) << "started a getAllShardsInfo(" << blkid.to_str() << ") liteserver query";
set_continuation([&]() -> void { continue_getAllShardsInfo(); });
request_mc_block_data_state(blkid);
request_mc_block_data(blkid);
}
void LiteQuery::continue_getShardInfo(ShardIdFull shard, bool exact) {
@ -1900,30 +2000,30 @@ void LiteQuery::continue_getShardInfo(ShardIdFull shard, bool exact) {
void LiteQuery::continue_getAllShardsInfo() {
LOG(INFO) << "completing getAllShardsInfo() query";
Ref<vm::Cell> proof1, proof2;
if (!make_mc_state_root_proof(proof1)) {
vm::MerkleProofBuilder mpb{mc_block_->root_cell()};
block::gen::Block::Record blk;
block::gen::BlockExtra::Record extra;
block::gen::McBlockExtra::Record mc_extra;
if (!tlb::unpack_cell(mpb.root(), blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() ||
!tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) {
fatal_error("cannot unpack header of block "s + mc_block_->block_id().to_str());
return;
}
vm::MerkleProofBuilder mpb{mc_state_->root_cell()};
auto shards_dict = block::ShardConfig::extract_shard_hashes_dict(mpb.root());
if (!shards_dict) {
fatal_error("cannot extract ShardHashes from last mc state");
return;
}
if (!mpb.extract_proof_to(proof2)) {
vm::Dictionary shards_dict(std::move(mc_extra.shard_hashes), 32);
Ref<vm::Cell> proof;
if (!mpb.extract_proof_to(proof)) {
fatal_error("cannot construct Merkle proof for all shards dictionary");
return;
}
shards_dict = block::ShardConfig::extract_shard_hashes_dict(mc_state_->root_cell());
vm::CellBuilder cb;
Ref<vm::Cell> cell;
if (!(std::move(shards_dict)->append_dict_to_bool(cb) && cb.finalize_to(cell))) {
fatal_error("cannot store ShardHashes from last mc state into a new cell");
auto proof_boc = vm::std_boc_serialize(std::move(proof));
if (proof_boc.is_error()) {
fatal_error(proof_boc.move_as_error());
return;
}
auto proof = vm::std_boc_serialize_multi({std::move(proof1), std::move(proof2)});
if (proof.is_error()) {
fatal_error(proof.move_as_error());
vm::CellBuilder cb;
Ref<vm::Cell> cell;
if (!(shards_dict.append_dict_to_bool(cb) && cb.finalize_to(cell))) {
fatal_error("cannot store ShardHashes from last mc block into a new cell");
return;
}
auto data = vm::std_boc_serialize(std::move(cell));
@ -1933,10 +2033,307 @@ void LiteQuery::continue_getAllShardsInfo() {
}
LOG(INFO) << "getAllShardInfo() query completed";
auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_allShardsInfo>(
ton::create_tl_lite_block_id(base_blk_id_), proof.move_as_ok(), data.move_as_ok());
ton::create_tl_lite_block_id(base_blk_id_), proof_boc.move_as_ok(), data.move_as_ok());
finish_query(std::move(b));
}
void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, int mode, LogicalTime lt, UnixTime utime) {
if (!((1 << (mode & 7)) & 0x16)) {
fatal_error("exactly one of mode.0, mode.1 and mode.2 bits must be set");
return;
}
if (!mc_blkid.is_masterchain_ext()) {
fatal_error("masterchain block id must be specified");
return;
}
if (!(mode & 1)) {
blkid.seqno = 0;
}
if (!(mode & 2)) {
lt = 0;
}
if (!(mode & 4)) {
utime = 0;
}
mode_ = mode;
base_blk_id_ = mc_blkid;
LOG(INFO) << "started a lookupBlockWithProof(" << blkid.to_str() << ", " << mc_blkid.to_str() << ", " << mode << ", "
<< lt << ", " << utime << ") liteserver query";
ton::AccountIdPrefixFull pfx{blkid.workchain, blkid.shard};
auto P = td::PromiseCreator::lambda(
[Self = actor_id(this), mc_blkid, manager = manager_, mode, pfx](td::Result<ConstBlockHandle> res) {
if (res.is_error()) {
td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
return;
}
auto handle = res.move_as_ok();
if (!handle->inited_masterchain_ref_block()) {
td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block doesn't have masterchain ref"));
return;
}
if (handle->masterchain_ref_block() > mc_blkid.seqno()) {
td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("specified mc block is older than block's masterchain ref"));
return;
}
LOG(DEBUG) << "requesting data for block " << handle->id().to_str();
td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle,
[Self, mc_ref_blkid = handle->masterchain_ref_block(), mc_blkid, pfx, mode](td::Result<Ref<BlockData>> res) {
if (res.is_error()) {
td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
} else {
td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_getHeaderProof, res.move_as_ok(), pfx, mc_ref_blkid);
}
});
});
if (mode & 2) {
td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_lt_from_db_for_litequery, pfx, lt, std::move(P));
} else if (mode & 4) {
td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_unix_time_from_db_for_litequery, pfx, utime, std::move(P));
} else {
td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_from_db_for_litequery, pfx, blkid.seqno, std::move(P));
}
}
void LiteQuery::continue_lookupBlockWithProof_getHeaderProof(Ref<ton::validator::BlockData> block, AccountIdPrefixFull req_prefix, BlockSeqno masterchain_ref_seqno) {
blk_id_ = block->block_id();
LOG(INFO) << "obtained data for getBlockHeader(" << blk_id_.to_str() << ", " << mode_ << ")";
CHECK(block.not_null());
auto block_root = block->root_cell();
if (block_root.is_null()) {
fatal_error("block has no valid root cell");
return;
}
vm::MerkleProofBuilder mpb{block_root};
std::vector<BlockIdExt> prev;
BlockIdExt mc_blkid;
bool after_split;
td::Status S = block::unpack_block_prev_blk_try(mpb.root(), blk_id_, prev, mc_blkid, after_split);
if (S.is_error()) {
fatal_error(std::move(S));
return;
}
auto proof_data = mpb.extract_proof_boc();
if (proof_data.is_error()) {
fatal_error(proof_data.move_as_error());
return;
}
lookup_header_proof_ = proof_data.move_as_ok();
bool include_prev = mode_ & 6;
if (include_prev) {
BlockIdExt prev_blkid;
for (auto& p : prev) {
if (ton::shard_contains(p.shard_full(), req_prefix)) {
prev_blkid = p;
}
}
CHECK(prev_blkid.is_valid());
get_block_handle_checked(prev_blkid, [Self = actor_id(this), masterchain_ref_seqno, manager = manager_](td::Result<ConstBlockHandle> R) mutable {
if (R.is_error()) {
td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error());
return;
}
td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(),
[Self, masterchain_ref_seqno](td::Result<Ref<BlockData>> res) mutable {
if (res.is_error()) {
td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
return;
}
td::actor::send_closure(Self, &LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData, res.move_as_ok(), masterchain_ref_seqno);
});
});
} else {
continue_lookupBlockWithProof_gotPrevBlockData(Ref<BlockData>(), masterchain_ref_seqno);
}
}
void LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData(Ref<BlockData> prev_block, BlockSeqno masterchain_ref_seqno) {
if (prev_block.not_null()) {
CHECK(prev_block.not_null());
if (prev_block->root_cell().is_null()) {
fatal_error("block has no valid root cell");
return;
}
vm::MerkleProofBuilder mpb{prev_block->root_cell()};
block::gen::Block::Record blk;
block::gen::BlockInfo::Record info;
if (!(tlb::unpack_cell(mpb.root(), blk) && tlb::unpack_cell(blk.info, info))) {
fatal_error(td::Status::Error("cannot unpack prev block header"));
return;
}
auto proof_data = mpb.extract_proof_boc();
if (proof_data.is_error()) {
fatal_error(proof_data.move_as_error());
return;
}
lookup_prev_header_proof_ = proof_data.move_as_ok();
}
if (!blk_id_.is_masterchain()) {
ton::AccountIdPrefixFull pfx{ton::masterchainId, ton::shardIdAll};
td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_from_db, pfx, masterchain_ref_seqno,
[manager = manager_, Self = actor_id(this)](td::Result<ConstBlockHandle> R) mutable {
if (R.is_error()) {
td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error());
return;
}
td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(),
[Self](td::Result<Ref<BlockData>> res) mutable {
if (res.is_error()) {
td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
return;
}
td::actor::send_closure(Self, &LiteQuery::continue_lookupBlockWithProof_buildProofLinks, res.move_as_ok(), std::vector<std::pair<BlockIdExt, td::Ref<vm::Cell>>>());
});
});
} else {
base_blk_id_alt_ = blk_id_;
td::actor::send_closure(actor_id(this), &LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState, std::vector<std::pair<BlockIdExt, Ref<vm::Cell>>>());
}
}
void LiteQuery::continue_lookupBlockWithProof_buildProofLinks(td::Ref<BlockData> cur_block,
std::vector<std::pair<BlockIdExt, td::Ref<vm::Cell>>> result) {
BlockIdExt cur_id = cur_block->block_id();
BlockIdExt prev_id;
vm::MerkleProofBuilder mpb{cur_block->root_cell()};
if (cur_id.is_masterchain()) {
base_blk_id_alt_ = cur_id;
block::gen::Block::Record blk;
block::gen::BlockExtra::Record extra;
block::gen::McBlockExtra::Record mc_extra;
if (!tlb::unpack_cell(mpb.root(), blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() ||
!tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) {
fatal_error("cannot unpack header of block "s + cur_id.to_str());
return;
}
block::ShardConfig shards(mc_extra.shard_hashes->prefetch_ref());
ShardIdFull shard_id = blk_id_.shard_full();
shard_id.shard = (shard_id.shard & ~(1 << (63 - shard_id.pfx_len()))) | 1;
Ref<block::McShardHash> shard_hash = shards.get_shard_hash(shard_id, false);
if (shard_hash.is_null()) {
fatal_error("shard not found");
return;
}
prev_id = shard_hash->top_block_id();
} else {
std::vector<BlockIdExt> prev;
BlockIdExt mc_blkid;
bool after_split;
td::Status S = block::unpack_block_prev_blk_try(mpb.root(), cur_id, prev, mc_blkid, after_split);
if (S.is_error()) {
fatal_error(std::move(S));
return;
}
bool found = false;
for (const BlockIdExt& id : prev) {
if (shard_intersects(id.shard_full(), blk_id_.shard_full())) {
found = true;
prev_id = id;
break;
}
}
if (!found) {
fatal_error("failed to find block chain");
return;
}
}
auto proof = mpb.extract_proof();
if (proof.is_error()) {
fatal_error(proof.move_as_error_prefix("cannot serialize Merkle proof : "));
return;
}
result.emplace_back(prev_id, proof.move_as_ok());
if (prev_id == blk_id_) {
CHECK(base_blk_id_alt_.is_masterchain());
if (base_blk_id_alt_ != base_blk_id_) {
continue_lookupBlockWithProof_getClientMcBlockDataState(std::move(result));
} else {
continue_lookupBlockWithProof_getMcBlockPrev(std::move(result));
}
return;
}
if (result.size() == 8) {
// Chains of shardblocks between masterchain blocks can't be longer than 8 (see collator.cpp:991)
fatal_error("proof chain is too long");
return;
}
td::actor::send_closure_later(
manager_, &ValidatorManager::get_block_data_from_db_short, prev_id,
[Self = actor_id(this), result = std::move(result)](td::Result<Ref<BlockData>> R) mutable {
if (R.is_error()) {
td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error());
} else {
td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_buildProofLinks, R.move_as_ok(),
std::move(result));
}
});
}
void LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState(std::vector<std::pair<BlockIdExt, td::Ref<vm::Cell>>> links) {
set_continuation([this, links = std::move(links)]() -> void {
continue_lookupBlockWithProof_getMcBlockPrev(std::move(links));
});
request_mc_block_data_state(base_blk_id_);
}
void LiteQuery::continue_lookupBlockWithProof_getMcBlockPrev(std::vector<std::pair<BlockIdExt, td::Ref<vm::Cell>>> links) {
td::BufferSlice mc_state_proof_buf, client_mc_blk_proof_buf;
if (base_blk_id_alt_ != base_blk_id_) {
vm::MerkleProofBuilder mpb{mc_state_->root_cell()};
auto prev_blocks_dict = block::get_prev_blocks_dict(mpb.root());
if (!prev_blocks_dict) {
fatal_error(td::Status::Error("cannot extract prev_blocks from mc state"));
return;
}
if (!block::check_old_mc_block_id(*prev_blocks_dict, base_blk_id_alt_)) {
fatal_error(td::Status::Error("client mc blkid is not in prev_blocks"));
return;
}
auto client_mc_blk_proof = mpb.extract_proof_boc();
if (client_mc_blk_proof.is_error()) {
fatal_error(client_mc_blk_proof.move_as_error());
return;
}
client_mc_blk_proof_buf = client_mc_blk_proof.move_as_ok();
Ref<vm::Cell> mc_state_proof;
if (!make_mc_state_root_proof(mc_state_proof)) {
fatal_error(td::Status::Error("cannot create Merkle proof for mc state"));
return;
}
auto mc_state_proof_boc = vm::std_boc_serialize(std::move(mc_state_proof));
if (mc_state_proof_boc.is_error()) {
fatal_error(mc_state_proof_boc.move_as_error());
return;
}
mc_state_proof_buf = mc_state_proof_boc.move_as_ok();
}
std::vector<tl_object_ptr<lite_api::liteServer_shardBlockLink>> links_res;
for (auto& p : links) {
auto prev_block_proof = vm::std_boc_serialize(std::move(p.second));
if (prev_block_proof.is_error()) {
fatal_error(prev_block_proof.move_as_error());
return;
}
links_res.push_back(
create_tl_object<lite_api::liteServer_shardBlockLink>(create_tl_lite_block_id(p.first), prev_block_proof.move_as_ok()));
}
auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_lookupBlockResult>(ton::create_tl_lite_block_id(blk_id_),
mode_, ton::create_tl_lite_block_id(base_blk_id_alt_), std::move(mc_state_proof_buf), std::move(client_mc_blk_proof_buf),
std::move(links_res), std::move(lookup_header_proof_), std::move(lookup_prev_header_proof_));
finish_query(std::move(b));
}
void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, UnixTime utime) {
if (!((1 << (mode & 7)) & 0x16)) {
fatal_error("exactly one of mode.0, mode.1 and mode.2 bits must be set");