From cf97f48cd74a7033a0e0613285da26227b7076c1 Mon Sep 17 00:00:00 2001 From: ton Date: Tue, 7 Apr 2020 00:08:53 +0400 Subject: [PATCH] error handling in lite client, speed up message dequeue in blocks --- crypto/block/block-parse.cpp | 21 +++-- crypto/block/block-parse.h | 6 +- crypto/block/block.tlb | 5 +- crypto/smartcont/CreateState.fif | 1 + crypto/smartcont/gen-zerostate.fif | 2 +- crypto/vm/cells/CellBuilder.h | 10 +-- lite-client/lite-client.cpp | 34 +++++++- ton/ton-types.h | 3 +- validator/db/archive-manager.cpp | 30 +++---- validator/db/archive-slice.cpp | 14 ++- validator/db/archive-slice.hpp | 12 ++- validator/impl/collator-impl.h | 2 + validator/impl/collator.cpp | 30 +++++-- validator/impl/liteserver.hpp | 6 +- validator/impl/validate-query.cpp | 136 +++++++++++++++++++++-------- 15 files changed, 224 insertions(+), 88 deletions(-) diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 22ac1b18..125d2a19 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -1844,9 +1844,13 @@ bool OutMsg::skip(vm::CellSlice& cs) const { && t_Ref_MsgEnvelope.skip(cs) // out_msg:^MsgEnvelope && RefTo{}.skip(cs); // reimport:^InMsg case msg_export_deq: - return cs.advance(3) // msg_export_deq$110 + return cs.advance(4) // msg_export_deq$1100 && t_Ref_MsgEnvelope.skip(cs) // out_msg:^MsgEnvelope - && cs.advance(64); // import_block_lt:uint64 + && cs.advance(63); // import_block_lt:uint63 + case msg_export_deq_short: + return cs.advance( + 4 + 256 + 32 + 64 + + 64); // msg_export_deq_short$1101 msg_env_hash:bits256 next_workchain:int32 next_addr_pfx:uint64 import_block_lt:uint64 case msg_export_tr_req: return cs.advance(3) // msg_export_tr_req$111 && t_Ref_MsgEnvelope.skip(cs) // out_msg:^MsgEnvelope @@ -1879,9 +1883,13 @@ bool OutMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // out_msg:^MsgEnvelope && RefTo{}.validate_skip(ops, cs, weak); // reimport:^InMsg case msg_export_deq: - return cs.advance(3) // msg_export_deq$110 + return cs.advance(4) // msg_export_deq$1100 && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // out_msg:^MsgEnvelope - && cs.advance(64); // import_block_lt:uint64 + && cs.advance(63); // import_block_lt:uint63 + case msg_export_deq_short: + return cs.advance( + 4 + 256 + 32 + 64 + + 64); // msg_export_deq_short$1101 msg_env_hash:bits256 next_workchain:int32 next_addr_pfx:uint64 import_block_lt:uint64 case msg_export_tr_req: return cs.advance(3) // msg_export_tr_req$111 && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // out_msg:^MsgEnvelope @@ -1902,7 +1910,9 @@ bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { case msg_export_deq_imm: // dequeuing record for outbound message delivered in this very block, no value exported return cs.have(3, 2) && t_CurrencyCollection.null_value(cb); case msg_export_deq: // dequeueing record for outbound message, no exported value - return cs.have(3, 1) && t_CurrencyCollection.null_value(cb); + return cs.have(4 + 63, 1) && t_CurrencyCollection.null_value(cb); + case msg_export_deq_short: // dequeueing record for outbound message, no exported value + return cs.have(4 + 256 + 32 + 64 + 64) && t_CurrencyCollection.null_value(cb); case msg_export_new: // newly-generated outbound internal message, queued case msg_export_tr: // transit internal message, queued case msg_export_tr_req: // transit internal message, re-queued from this shardchain @@ -1941,6 +1951,7 @@ bool OutMsg::get_created_lt(vm::CellSlice& cs, unsigned long long& created_lt) c case msg_export_new: case msg_export_tr: case msg_export_deq: + case msg_export_deq_short: case msg_export_deq_imm: case msg_export_tr_req: if (cs.have(3, 1)) { diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index e1578df0..b2ed9c95 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -811,13 +811,15 @@ struct OutMsg final : TLB_Complex { msg_export_imm = 2, msg_export_tr = 3, msg_export_deq_imm = 4, - msg_export_deq = 6, + msg_export_deq = 12, + msg_export_deq_short = 13, msg_export_tr_req = 7 }; bool skip(vm::CellSlice& cs) const override; bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; int get_tag(const vm::CellSlice& cs) const override { - return (int)cs.prefetch_ulong(3); + int t = (int)cs.prefetch_ulong(3); + return t != 6 ? t : (int)cs.prefetch_ulong(4); } bool get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const; bool get_created_lt(vm::CellSlice& cs, unsigned long long& created_lt) const; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 86e7e5b8..ee85d6f6 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -193,7 +193,10 @@ msg_export_new$001 out_msg:^MsgEnvelope transaction:^Transaction = OutMsg; msg_export_tr$011 out_msg:^MsgEnvelope imported:^InMsg = OutMsg; -msg_export_deq$110 out_msg:^MsgEnvelope // out_msg_hash:bits256 ? +msg_export_deq$1100 out_msg:^MsgEnvelope + import_block_lt:uint63 = OutMsg; +msg_export_deq_short$1101 msg_env_hash:bits256 + next_workchain:int32 next_addr_pfx:uint64 import_block_lt:uint64 = OutMsg; msg_export_tr_req$111 out_msg:^MsgEnvelope imported:^InMsg = OutMsg; diff --git a/crypto/smartcont/CreateState.fif b/crypto/smartcont/CreateState.fif index 2779073b..1022b26e 100644 --- a/crypto/smartcont/CreateState.fif +++ b/crypto/smartcont/CreateState.fif @@ -43,6 +43,7 @@ 4 constant capBounceMsgBody 8 constant capReportVersion 16 constant capSplitMergeTransactions +32 constant capShortDequeue // max-validators masterchain-validators min-validators -- { swap rot 16 config! } : config.validator_num! diff --git a/crypto/smartcont/gen-zerostate.fif b/crypto/smartcont/gen-zerostate.fif index b0d1f4ea..1dfe8411 100644 --- a/crypto/smartcont/gen-zerostate.fif +++ b/crypto/smartcont/gen-zerostate.fif @@ -157,7 +157,7 @@ Masterchain swap * */ // version capabilities -1 capCreateStats capBounceMsgBody or capReportVersion or config.version! +1 capCreateStats capBounceMsgBody or capReportVersion or capShortDeque or config.version! // max-validators max-main-validators min-validators // 9 4 1 config.validator_num! 1000 100 13 config.validator_num! diff --git a/crypto/vm/cells/CellBuilder.h b/crypto/vm/cells/CellBuilder.h index 4efd90d8..954a1ac0 100644 --- a/crypto/vm/cells/CellBuilder.h +++ b/crypto/vm/cells/CellBuilder.h @@ -107,9 +107,9 @@ class CellBuilder : public td::CntObject { CellBuilder& store_bits_same(std::size_t bit_count, bool val); bool store_bits_bool(const unsigned char* str, std::size_t bit_count, int bit_offset = 0); bool store_bits_bool(td::ConstBitPtr bs, std::size_t bit_count); - template - bool store_bits_bool(const td::BitArray& ba) { - return store_bits_bool(ba.cbits(), n); + template + bool store_bits_bool(const T& ba) { + return store_bits_bool(ba.bits(), ba.size()); } bool store_bits_same_bool(std::size_t bit_count, bool val); CellBuilder& store_zeroes(std::size_t bit_count) { @@ -181,9 +181,9 @@ class CellBuilder : public td::CntObject { bool finalize_to(Ref& res, bool special = false) { return (res = finalize(special)).not_null(); } - CellSlice as_cellslice() const&; + CellSlice as_cellslice() const &; CellSlice as_cellslice() &&; - Ref as_cellslice_ref() const&; + Ref as_cellslice_ref() const &; Ref as_cellslice_ref() &&; static td::int64 get_total_cell_builders() { return get_thread_safe_counter().sum(); diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index ad0cae9d..69eadb05 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -1711,9 +1711,13 @@ void TestNode::run_smc_method(int mode, ton::BlockIdExt ref_blk, ton::BlockIdExt case block::gen::AccountState::account_uninit: LOG(ERROR) << "account " << workchain << ":" << addr.to_hex() << " not initialized yet (cannot run any methods)"; + promise.set_error(td::Status::Error(PSLICE() << "account " << workchain << ":" << addr.to_hex() + << " not initialized yet (cannot run any methods)")); return; case block::gen::AccountState::account_frozen: LOG(ERROR) << "account " << workchain << ":" << addr.to_hex() << " frozen (cannot run any methods)"; + promise.set_error(td::Status::Error(PSLICE() << "account " << workchain << ":" << addr.to_hex() + << " frozen (cannot run any methods)")); return; } CHECK(store.state.write().fetch_ulong(1) == 1); // account_init$1 _:StateInit = AccountState; @@ -1730,7 +1734,7 @@ void TestNode::run_smc_method(int mode, ton::BlockIdExt ref_blk, ton::BlockIdExt stack->dump(os, 3); out << os.str(); } - long long gas_limit = vm::GasLimits::infty; + long long gas_limit = /* vm::GasLimits::infty */ 10000000; // OstreamLogger ostream_logger(ctx.error_stream); // auto log = create_vm_log(ctx.error_stream ? &ostream_logger : nullptr); vm::GasLimits gas{gas_limit}; @@ -1741,10 +1745,27 @@ void TestNode::run_smc_method(int mode, ton::BlockIdExt ref_blk, ton::BlockIdExt // vm.incr_stack_trace(1); // enable stack dump after each step LOG(INFO) << "starting VM to run method `" << method << "` (" << method_id << ") of smart contract " << workchain << ":" << addr.to_hex(); - int exit_code = ~vm.run(); + int exit_code; + try { + exit_code = ~vm.run(); + } catch (vm::VmVirtError& err) { + LOG(ERROR) << "virtualization error while running VM to locally compute runSmcMethod result: " << err.get_msg(); + promise.set_error( + td::Status::Error(PSLICE() << "virtualization error while running VM to locally compute runSmcMethod result: " + << err.get_msg())); + exit_code = -1001; + } catch (vm::VmError& err) { + LOG(ERROR) << "error while running VM to locally compute runSmcMethod result: " << err.get_msg(); + promise.set_error(td::Status::Error(PSLICE() << "error while running VM to locally compute runSmcMethod result: " + << err.get_msg())); + exit_code = -1000; + } LOG(DEBUG) << "VM terminated with exit code " << exit_code; if (mode > 0) { LOG(DEBUG) << "remote VM exit code is " << remote_exit_code; + if (remote_exit_code == ~(int)vm::Excno::out_of_gas) { + LOG(WARNING) << "remote server ran out of gas while performing this request; consider using runmethodfull"; + } } if (exit_code != 0) { LOG(ERROR) << "VM terminated with error code " << exit_code; @@ -1765,13 +1786,17 @@ void TestNode::run_smc_method(int mode, ton::BlockIdExt ref_blk, ton::BlockIdExt } else { auto res = vm::std_boc_deserialize(std::move(remote_result)); if (res.is_error()) { - LOG(ERROR) << "cannot deserialize remote VM result boc: " << res.move_as_error(); + auto err = res.move_as_error(); + LOG(ERROR) << "cannot deserialize remote VM result boc: " << err; + promise.set_error( + td::Status::Error(PSLICE() << "cannot deserialize remote VM result boc: " << std::move(err))); return; } auto cs = vm::load_cell_slice(res.move_as_ok()); Ref remote_stack; if (!(vm::Stack::deserialize_to(cs, remote_stack, 0) && cs.empty_ext())) { LOG(ERROR) << "remote VM result boc cannot be deserialized as a VmStack"; + promise.set_error(td::Status::Error("remote VM result boc cannot be deserialized as a VmStack")); return; } std::ostringstream os; @@ -1784,8 +1809,11 @@ void TestNode::run_smc_method(int mode, ton::BlockIdExt ref_blk, ton::BlockIdExt promise.set_result(stack->extract_contents()); } catch (vm::VmVirtError& err) { out << "virtualization error while parsing runSmcMethod result: " << err.get_msg(); + promise.set_error( + td::Status::Error(PSLICE() << "virtualization error while parsing runSmcMethod result: " << err.get_msg())); } catch (vm::VmError& err) { out << "error while parsing runSmcMethod result: " << err.get_msg(); + promise.set_error(td::Status::Error(PSLICE() << "error while parsing runSmcMethod result: " << err.get_msg())); } } diff --git a/ton/ton-types.h b/ton/ton-types.h index 09e95059..a5163ec5 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -61,7 +61,8 @@ enum GlobalCapabilities { capCreateStatsEnabled = 2, capBounceMsgBody = 4, capReportVersion = 8, - capSplitMergeTransactions = 16 + capSplitMergeTransactions = 16, + capShortDequeue = 32 }; inline int shard_pfx_len(ShardId shard) { diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index c5f5ba9d..d9bd3f01 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -98,12 +98,13 @@ void ArchiveManager::add_file(BlockHandle handle, FileReference ref_id, td::Buff auto ig = mp.init_guard(); ig.add_promise(std::move(promise)); auto f1 = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true); - td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), data.clone(), + td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, nullptr, std::move(ref_id), data.clone(), ig.get_promise()); if (copy_to_key) { auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()), handle->id().seqno(), handle->unix_time(), handle->logical_time(), true); - td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, ref_id, std::move(data), ig.get_promise()); + td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, nullptr, ref_id, std::move(data), + ig.get_promise()); } return; } @@ -115,24 +116,25 @@ void ArchiveManager::add_file(BlockHandle handle, FileReference ref_id, td::Buff ig.add_promise(std::move(promise)); auto f1 = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), handle->id().seqno(), handle->unix_time(), handle->logical_time(), true); - td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, ref_id, data.clone(), ig.get_promise()); + td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, handle, ref_id, data.clone(), ig.get_promise()); if (copy_to_key) { auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()), handle->id().seqno(), handle->unix_time(), handle->logical_time(), true); - td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, ref_id, std::move(data), ig.get_promise()); + td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, handle, ref_id, std::move(data), + ig.get_promise()); } } void ArchiveManager::add_key_block_proof(UnixTime ts, BlockSeqno seqno, LogicalTime lt, FileReference ref_id, td::BufferSlice data, td::Promise promise) { auto f = get_file_desc(ShardIdFull{masterchainId}, get_key_package_id(seqno), seqno, ts, lt, true); - td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), std::move(data), + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, nullptr, std::move(ref_id), std::move(data), std::move(promise)); } void ArchiveManager::add_temp_file_short(FileReference ref_id, td::BufferSlice data, td::Promise promise) { auto f = get_file_desc(ref_id.shard(), get_temp_package_id(), 0, 0, 0, true); - td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), std::move(data), + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, nullptr, std::move(ref_id), std::move(data), std::move(promise)); } @@ -537,7 +539,7 @@ void ArchiveManager::load_package(PackageId id) { } } - desc.file = td::actor::create_actor("slice", id.key, id.temp, prefix); + desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, prefix); get_file_map(id).emplace(id, std::move(desc)); } @@ -571,7 +573,7 @@ ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard FileDescription desc{id, false}; td::mkdir(db_root_ + id.path()).ensure(); std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); - desc.file = td::actor::create_actor("slice", id.key, id.temp, prefix); + desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, prefix); if (!id.temp) { update_desc(desc, shard, seqno, ts, lt); } @@ -992,23 +994,19 @@ void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, td::Promiseid.id); + td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_archive_id, masterchain_seqno, std::move(promise)); } void ArchiveManager::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) { - if (archive_id != static_cast(archive_id)) { - promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found")); - return; - } - auto F = get_file_desc(ShardIdFull{masterchainId}, PackageId{static_cast(archive_id), false, false}, 0, 0, - 0, false); + auto arch = static_cast(archive_id); + auto F = get_file_desc(ShardIdFull{masterchainId}, PackageId{arch, false, false}, 0, 0, 0, false); if (!F) { promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found")); return; } - td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_slice, offset, limit, std::move(promise)); + td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_slice, archive_id, offset, limit, std::move(promise)); } void ArchiveManager::commit_transaction() { diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index aad9c8fa..3865d25c 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -162,7 +162,8 @@ void ArchiveSlice::update_handle(BlockHandle handle, td::Promise promi promise.set_value(td::Unit()); } -void ArchiveSlice::add_file(FileReference ref_id, td::BufferSlice data, td::Promise promise) { +void ArchiveSlice::add_file(BlockHandle handle, FileReference ref_id, td::BufferSlice data, + td::Promise promise) { if (destroyed_) { promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); return; @@ -409,7 +410,12 @@ td::BufferSlice ArchiveSlice::get_db_key_block_info(BlockIdExt block_id) { return create_serialize_tl_object(create_tl_block_id(block_id)); } -void ArchiveSlice::get_slice(td::uint64 offset, td::uint32 limit, td::Promise promise) { +void ArchiveSlice::get_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, + td::Promise promise) { + if (archive_id != archive_id_) { + promise.set_error(td::Status::Error(ErrorCode::error, "bad archive id")); + return; + } td::actor::create_actor("readfile", prefix_ + ".pack", offset, limit, 0, std::move(promise)).release(); } @@ -467,8 +473,8 @@ void ArchiveSlice::set_async_mode(bool mode, td::Promise promise) { td::actor::send_closure(writer_, &PackageWriter::set_async_mode, mode, std::move(promise)); } -ArchiveSlice::ArchiveSlice(bool key_blocks_only, bool temp, std::string prefix) - : key_blocks_only_(key_blocks_only), temp_(temp), prefix_(std::move(prefix)) { +ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, std::string prefix) + : archive_id_(archive_id), key_blocks_only_(key_blocks_only), temp_(temp), prefix_(std::move(prefix)) { } namespace { diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index dc7b2341..65f323ce 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -47,11 +47,15 @@ class PackageWriter : public td::actor::Actor { class ArchiveSlice : public td::actor::Actor { public: - ArchiveSlice(bool key_blocks_only, bool temp, std::string prefix); + ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, std::string prefix); + + void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { + promise.set_result(archive_id_); + } void add_handle(BlockHandle handle, td::Promise promise); void update_handle(BlockHandle handle, td::Promise promise); - void add_file(FileReference ref_id, td::BufferSlice data, td::Promise promise); + void add_file(BlockHandle handle, FileReference ref_id, td::BufferSlice data, td::Promise promise); void get_handle(BlockIdExt block_id, td::Promise promise); void get_temp_handle(BlockIdExt block_id, td::Promise promise); void get_file(FileReference ref_id, td::Promise promise); @@ -65,7 +69,7 @@ class ArchiveSlice : public td::actor::Actor { std::function compare, bool exact, td::Promise promise); - void get_slice(td::uint64 offset, td::uint32 limit, td::Promise promise); + void get_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise); void start_up() override; void destroy(td::Promise promise); @@ -84,6 +88,8 @@ class ArchiveSlice : public td::actor::Actor { td::BufferSlice get_db_key_block_info(BlockIdExt block_id); td::BufferSlice get_lt_from_db(ShardIdFull shard, td::uint32 idx); + td::uint32 archive_id_; + bool key_blocks_only_; bool temp_; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index dad2fe8b..12f46188 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -147,6 +147,7 @@ class Collator final : public td::actor::Actor { bool report_version_{false}; bool skip_topmsgdescr_{false}; bool skip_extmsg_{false}; + bool short_dequeue_records_{false}; td::uint64 overload_history_{0}, underload_history_{0}; td::uint64 block_size_estimate_{}; Ref wc_info_; @@ -275,6 +276,7 @@ class Collator final : public td::actor::Actor { bool delete_out_msg_queue_msg(td::ConstBitPtr key); bool insert_in_msg(Ref in_msg); bool insert_out_msg(Ref out_msg); + bool insert_out_msg(Ref out_msg, td::ConstBitPtr msg_hash); bool register_out_msg_queue_op(bool force = false); bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno); bool combine_account_transactions(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index f4f2d869..1d08089e 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -512,6 +512,7 @@ bool Collator::unpack_last_mc_state() { ihr_enabled_ = config_->ihr_enabled(); create_stats_enabled_ = config_->create_stats_enabled(); report_version_ = config_->has_capability(ton::capReportVersion); + short_dequeue_records_ = config_->has_capability(ton::capShortDequeue); shard_conf_ = std::make_unique(*config_); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); if (prev_key_block_exists_) { @@ -1770,10 +1771,20 @@ bool Collator::do_collate() { bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime delivered_lt) { LOG(DEBUG) << "dequeueing outbound message"; vm::CellBuilder cb; - return cb.store_long_bool(6, 3) // msg_export_deq$110 - && cb.store_ref_bool(msg_envelope) // out_msg:^MsgEnvelope - && cb.store_long_bool(delivered_lt, 64) // import_block_lt:uint64 - && insert_out_msg(cb.finalize()); + if (short_dequeue_records_) { + td::BitArray<352> out_queue_key; + return block::compute_out_msg_queue_key(msg_envelope, out_queue_key) // (compute key) + && cb.store_long_bool(13, 4) // msg_export_deq_short$1101 + && cb.store_bits_bool(msg_envelope->get_hash().as_bitslice()) // msg_env_hash:bits256 + && cb.store_bits_bool(out_queue_key.bits(), 96) // next_workchain:int32 next_addr_pfx:uint64 + && cb.store_long_bool(delivered_lt, 64) // import_block_lt:uint64 + && insert_out_msg(cb.finalize(), out_queue_key.bits() + 96); + } else { + return cb.store_long_bool(12, 4) // msg_export_deq$1100 + && cb.store_ref_bool(msg_envelope) // out_msg:^MsgEnvelope + && cb.store_long_bool(delivered_lt, 63) // import_block_lt:uint63 + && insert_out_msg(cb.finalize()); + } } bool Collator::out_msg_queue_cleanup() { @@ -2783,7 +2794,7 @@ bool Collator::insert_in_msg(Ref in_msg) { // inserts an OutMsg into OutMsgDescr bool Collator::insert_out_msg(Ref out_msg) { if (verbosity > 2) { - fprintf(stderr, "OutMsg being inserted into OutMsgDescr: "); + std::cerr << "OutMsg being inserted into OutMsgDescr: "; block::gen::t_OutMsg.print_ref(std::cerr, out_msg); } auto cs = load_cell_slice(out_msg); @@ -2800,10 +2811,14 @@ bool Collator::insert_out_msg(Ref out_msg) { } msg = cs2.prefetch_ref(); // use hash of (Message Any) } + return insert_out_msg(std::move(out_msg), msg->get_hash().bits()); +} + +bool Collator::insert_out_msg(Ref out_msg, td::ConstBitPtr msg_hash) { bool ok; try { - ok = out_msg_dict->set(msg->get_hash().bits(), 256, cs, vm::Dictionary::SetMode::Add); - } catch (vm::VmError) { + ok = out_msg_dict->set(msg_hash, 256, load_cell_slice(std::move(out_msg)), vm::Dictionary::SetMode::Add); + } catch (vm::VmError&) { ok = false; } if (!ok) { @@ -2814,6 +2829,7 @@ bool Collator::insert_out_msg(Ref out_msg) { return block_limit_status_->add_cell(std::move(out_msg)) && ((out_descr_cnt_ & 63) || block_limit_status_->add_cell(out_msg_dict->get_root_cell())); } + // enqueues a new Message into OutMsgDescr and OutMsgQueue bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_remaining, ton::LogicalTime enqueued_lt) { // 0. unpack src_addr and dest_addr diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index bcdc5050..f1922b3b 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -62,7 +62,7 @@ class LiteQuery : public td::actor::Actor { enum { default_timeout_msec = 4500, // 4.5 seconds max_transaction_count = 16, // fetch at most 16 transactions in one query - client_method_gas_limit = 100000 // gas limit for liteServer.runSmcMethod + client_method_gas_limit = 300000 // gas limit for liteServer.runSmcMethod }; enum { ls_version = 0x101, @@ -160,8 +160,8 @@ class LiteQuery : public td::actor::Actor { const BlockIdExt& blkid); bool make_state_root_proof(Ref& proof, Ref state_root, Ref block_root, const BlockIdExt& blkid); - bool make_shard_info_proof(Ref& proof, Ref& info, ShardIdFull shard, ShardIdFull& true_shard, - Ref& leaf, bool& found, bool exact = true); + bool make_shard_info_proof(Ref& proof, Ref& info, ShardIdFull shard, + ShardIdFull& true_shard, Ref& leaf, bool& found, bool exact = true); bool make_shard_info_proof(Ref& proof, Ref& info, ShardIdFull shard, bool exact = true); bool make_shard_info_proof(Ref& proof, Ref& info, AccountIdPrefixFull prefix); bool make_shard_info_proof(Ref& proof, BlockIdExt& blkid, AccountIdPrefixFull prefix); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 711c2e8c..02316231 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -2609,7 +2609,7 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id } auto q_msg_env = (old_value.not_null() ? old_value : new_value)->prefetch_ref(); int tag = (int)out_msg_cs->prefetch_ulong(3); - // mode for msg_export_{ext,new,imm,tr,deq_imm,???,deq,tr_req} + // mode for msg_export_{ext,new,imm,tr,deq_imm,???,deq/deq_short,tr_req} static const int tag_mode[8] = {0, 2, 0, 2, 1, 0, 1, 3}; static const char* tag_str[8] = {"ext", "new", "imm", "tr", "deq_imm", "???", "deq", "tr_req"}; if (tag < 0 || tag >= 8 || !(tag_mode[tag] & mode)) { @@ -2617,18 +2617,33 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id << out_msg_id.to_hex(352) << " has invalid tag " << tag << "(" << tag_str[tag & 7] << ")"); } - auto msg_env = out_msg_cs->prefetch_ref(); - if (msg_env.is_null()) { - return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) + " is invalid"); - } - auto msg = vm::load_cell_slice(msg_env).prefetch_ref(); - if (msg.is_null()) { - return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) + " is invalid"); - } - if (msg->get_hash().as_bitslice() != out_msg_id + 96) { - return reject_query("OutMsgDescr for "s + (out_msg_id + 96).to_hex(256) + - " contains a message with different hash "s + msg->get_hash().bits().to_hex(256)); + bool is_short = (tag == 6 && (out_msg_cs->prefetch_ulong(4) & + 1)); // msg_export_deq_short does not contain true MsgEnvelope / Message + Ref msg_env, msg; + td::Bits256 msg_env_hash; + block::gen::OutMsg::Record_msg_export_deq_short deq_short; + if (!is_short) { + msg_env = out_msg_cs->prefetch_ref(); + if (msg_env.is_null()) { + return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) + " is invalid (contains no MsgEnvelope)"); + } + msg_env_hash = msg_env->get_hash().bits(); + msg = vm::load_cell_slice(msg_env).prefetch_ref(); + if (msg.is_null()) { + return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) + " is invalid (contains no message)"); + } + if (msg->get_hash().as_bitslice() != out_msg_id + 96) { + return reject_query("OutMsgDescr for "s + (out_msg_id + 96).to_hex(256) + + " contains a message with different hash "s + msg->get_hash().bits().to_hex(256)); + } + } else { + if (!tlb::csr_unpack(out_msg_cs, deq_short)) { // parsing msg_export_deq_short$1101 ... + return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) + + " is invalid (cannot unpack msg_export_deq_short)"); + } + msg_env_hash = deq_short.msg_env_hash; } + // if (mode == 1) { // dequeued message if (tag == 7) { @@ -2665,13 +2680,13 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id " is a msg_export_tr_req referring to a reimport InMsgDescr that contains a MsgEnvelope " "distinct from that originally kept in the old queue"); } - } else if (msg_env->get_hash().as_bitslice() != q_msg_env->get_hash().bits()) { + } else if (msg_env_hash != q_msg_env->get_hash().bits()) { return reject_query("OutMsgDescr corresponding to dequeued message with key "s + out_msg_id.to_hex(352) + " contains a MsgEnvelope distinct from that originally kept in the old queue"); } } else { // enqueued message - if (msg_env->get_hash().as_bitslice() != q_msg_env->get_hash().bits()) { + if (msg_env_hash != q_msg_env->get_hash().bits()) { return reject_query("OutMsgDescr corresponding to "s + m_str[mode] + "queued message with key "s + out_msg_id.to_hex(352) + " contains a MsgEnvelope distinct from that stored in the new queue"); @@ -2680,14 +2695,24 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id // in all cases above, we have to check that all 352-bit key is correct (including first 96 bits) // otherwise we might not be able to correctly recover OutMsgQueue entries starting from OutMsgDescr later // or we might have several OutMsgQueue entries with different 352-bit keys all having the same last 256 bits (with the message hash) + if (is_short) { + // check out_msg_id using fields next_workchain:int32 next_addr_pfx:uint64 of msg_export_deq_short$1101 + if (out_msg_id.get_int(32) != deq_short.next_workchain || + (out_msg_id + 32).get_uint(64) != deq_short.next_addr_pfx) { + return reject_query( + PSTRING() << "OutMsgQueue entry with key " << out_msg_id.to_hex(352) + << " corresponds to msg_export_deq_short OutMsg entry with incorrect next hop parameters " + << deq_short.next_workchain << "," << deq_short.next_addr_pfx); + } + } td::BitArray<352> key; if (!block::compute_out_msg_queue_key(q_msg_env, key)) { - return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) + + return reject_query("OutMsgQueue entry with key "s + out_msg_id.to_hex(352) + " refers to a MsgEnvelope that cannot be unpacked"); } if (key != out_msg_id) { - return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) + - " contains a MsgEnvelope that should be stored under different key " + key.to_hex()); + return reject_query("OutMsgQueue entry with key "s + out_msg_id.to_hex(352) + + " contains a MsgEnvelope that should have been stored under different key " + key.to_hex()); } return true; } @@ -2732,7 +2757,6 @@ bool ValidateQuery::update_min_enqueued_lt_hash(ton::LogicalTime lt, const ton:: // check that the enveloped message (MsgEnvelope) was present in the output queue of a neighbor, and that it has not been processed before bool ValidateQuery::check_imported_message(Ref msg_env) { - // TODO block::tlb::MsgEnvelope::Record_std env; block::gen::CommonMsgInfo::Record_int_msg_info info; ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix; @@ -3231,6 +3255,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms Ref src, dest; Ref transaction; Ref msg, msg_env, tr_msg_env, reimport; + td::Bits256 msg_env_hash; // msg_envelope#4 cur_addr:IntermediateAddress next_addr:IntermediateAddress fwd_fee_remaining:Grams msg:^(Message Any) = MsgEnvelope; block::tlb::MsgEnvelope::Record_std env; // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool @@ -3243,6 +3268,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms ton::LogicalTime import_lt = ~0ULL; unsigned long long created_lt = 0; int mode = 0, in_tag = -2; + bool is_short = false; // initial checks and unpack switch (tag) { case block::gen::OutMsg::msg_export_ext: { @@ -3317,6 +3343,18 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms // ... break; } + case block::gen::OutMsg::msg_export_deq_short: { + block::gen::OutMsg::Record_msg_export_deq_short out; + CHECK(tlb::csr_unpack(out_msg, out)); + msg_env_hash = out.msg_env_hash; + next_prefix.workchain = out.next_workchain; + next_prefix.account_id_prefix = out.next_addr_pfx; + import_lt = out.import_block_lt; + is_short = true; + mode = 1; // removed from OutMsgQueue + // ... + break; + } case block::gen::OutMsg::msg_export_tr_req: { block::gen::OutMsg::Record_msg_export_tr_req out; CHECK(tlb::csr_unpack(out_msg, out) && tlb::unpack_cell(out.out_msg, env)); @@ -3343,15 +3381,22 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms return reject_query(PSTRING() << "OutMsg with key (message hash) " << key.to_hex(256) << " has an unknown tag " << tag); } - - // common checks for all (non-external) inbound messages - CHECK(msg.not_null()); - if (msg->get_hash().as_bitslice() != key) { - return reject_query("OutMsg with key "s + key.to_hex(256) + " refers to a message with different hash " + - msg->get_hash().to_hex()); + if (msg_env.not_null()) { + msg_env_hash = msg_env->get_hash().bits(); } - if (tag != block::gen::OutMsg::msg_export_ext) { + // common checks for all (non-external) outbound messages + if (!is_short) { + CHECK(msg.not_null()); + if (msg->get_hash().as_bitslice() != key) { + return reject_query("OutMsg with key "s + key.to_hex(256) + " refers to a message with different hash " + + msg->get_hash().to_hex()); + } + } + + if (is_short) { + // nothing to check here for msg_export_deq_short ? + } else if (tag != block::gen::OutMsg::msg_export_ext) { // unpack int_msg_info$0 ... = CommonMsgInfo, especially message addresses if (!tlb::unpack_cell_inexact(msg, info)) { return reject_query("OutMsg with key "s + key.to_hex(256) + @@ -3447,7 +3492,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms td::BitArray<32 + 64 + 256> q_key; q_key.bits().store_int(next_prefix.workchain, 32); (q_key.bits() + 32).store_int(next_prefix.account_id_prefix, 64); - (q_key.bits() + 96).copy_from(env.msg->get_hash().bits(), 256); + (q_key.bits() + 96).copy_from(key, 256); auto q_entry = ns_.out_msg_queue_->lookup(q_key); auto old_q_entry = ps_.out_msg_queue_->lookup(q_key); if (old_q_entry.not_null() && q_entry.not_null()) { @@ -3471,7 +3516,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms return reject_query("OutMsg with key "s + key.to_hex(256) + " was expected to create OutMsgQueue entry with key " + q_key.to_hex() + " but it did not"); } - if (q_entry->prefetch_ref()->get_hash() != msg_env->get_hash()) { + if (msg_env_hash != q_entry->prefetch_ref()->get_hash().bits()) { return reject_query("OutMsg with key "s + key.to_hex(256) + " has created OutMsgQueue entry with key " + q_key.to_hex() + " containing a different MsgEnvelope"); } @@ -3482,7 +3527,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms " was expected to remove OutMsgQueue entry with key " + q_key.to_hex() + " but it did not exist in the old queue"); } - if (old_q_entry->prefetch_ref()->get_hash() != msg_env->get_hash()) { + if (msg_env_hash != old_q_entry->prefetch_ref()->get_hash().bits()) { return reject_query("OutMsg with key "s + key.to_hex(256) + " has dequeued OutMsgQueue entry with key " + q_key.to_hex() + " containing a different MsgEnvelope"); } @@ -3599,7 +3644,8 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms // ... break; } - case block::gen::OutMsg::msg_export_deq: { + case block::gen::OutMsg::msg_export_deq: + case block::gen::OutMsg::msg_export_deq_short: { // check that the message has been indeed processed by a neighbor CHECK(old_q_entry.not_null()); block::EnqueuedMsgDescr enq_msg_descr; @@ -3841,14 +3887,30 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, " already processed by this shard, but there is no ext_message_deq OutMsg record for this " "message in this block"); } - block::gen::OutMsg::Record_msg_export_deq deq; - if (!tlb::csr_unpack(std::move(out_entry), deq)) { - return reject_query("cannot unpack msg_export_deq OutMsg record for already processed EnqueuedMsg with key "s + - key.to_hex(352) + " of old outbound queue"); - } - if (deq.out_msg->get_hash() != enq.msg_env_->get_hash()) { - return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s + - key.to_hex(352) + " of old outbound queue contains a different MsgEnvelope"); + int tag = block::gen::t_OutMsg.get_tag(*out_entry); + if (tag == block::gen::OutMsg::msg_export_deq_short) { + block::gen::OutMsg::Record_msg_export_deq_short deq; + if (!tlb::csr_unpack(std::move(out_entry), deq)) { + return reject_query( + "cannot unpack msg_export_deq_short OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue"); + } + if (deq.msg_env_hash != enq.msg_env_->get_hash().bits()) { + return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue refers to MsgEnvelope with different hash " + + deq.msg_env_hash.to_hex()); + } + } else { + block::gen::OutMsg::Record_msg_export_deq deq; + if (!tlb::csr_unpack(std::move(out_entry), deq)) { + return reject_query( + "cannot unpack msg_export_deq OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue"); + } + if (deq.out_msg->get_hash() != enq.msg_env_->get_hash()) { + return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue contains a different MsgEnvelope"); + } } } // next check is incorrect after a merge, when ns_.processed_upto has > 1 entries