1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00

error handling in lite client, speed up message dequeue in blocks

This commit is contained in:
ton 2020-04-07 00:08:53 +04:00
parent dfc040cb00
commit cf97f48cd7
15 changed files with 224 additions and 88 deletions

View file

@ -147,6 +147,7 @@ class Collator final : public td::actor::Actor {
bool report_version_{false};
bool skip_topmsgdescr_{false};
bool skip_extmsg_{false};
bool short_dequeue_records_{false};
td::uint64 overload_history_{0}, underload_history_{0};
td::uint64 block_size_estimate_{};
Ref<block::WorkchainInfo> wc_info_;
@ -275,6 +276,7 @@ class Collator final : public td::actor::Actor {
bool delete_out_msg_queue_msg(td::ConstBitPtr key);
bool insert_in_msg(Ref<vm::Cell> in_msg);
bool insert_out_msg(Ref<vm::Cell> out_msg);
bool insert_out_msg(Ref<vm::Cell> out_msg, td::ConstBitPtr msg_hash);
bool register_out_msg_queue_op(bool force = false);
bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno);
bool combine_account_transactions();

View file

@ -512,6 +512,7 @@ bool Collator::unpack_last_mc_state() {
ihr_enabled_ = config_->ihr_enabled();
create_stats_enabled_ = config_->create_stats_enabled();
report_version_ = config_->has_capability(ton::capReportVersion);
short_dequeue_records_ = config_->has_capability(ton::capShortDequeue);
shard_conf_ = std::make_unique<block::ShardConfig>(*config_);
prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_);
if (prev_key_block_exists_) {
@ -1770,10 +1771,20 @@ bool Collator::do_collate() {
bool Collator::dequeue_message(Ref<vm::Cell> msg_envelope, ton::LogicalTime delivered_lt) {
LOG(DEBUG) << "dequeueing outbound message";
vm::CellBuilder cb;
return cb.store_long_bool(6, 3) // msg_export_deq$110
&& cb.store_ref_bool(msg_envelope) // out_msg:^MsgEnvelope
&& cb.store_long_bool(delivered_lt, 64) // import_block_lt:uint64
&& insert_out_msg(cb.finalize());
if (short_dequeue_records_) {
td::BitArray<352> out_queue_key;
return block::compute_out_msg_queue_key(msg_envelope, out_queue_key) // (compute key)
&& cb.store_long_bool(13, 4) // msg_export_deq_short$1101
&& cb.store_bits_bool(msg_envelope->get_hash().as_bitslice()) // msg_env_hash:bits256
&& cb.store_bits_bool(out_queue_key.bits(), 96) // next_workchain:int32 next_addr_pfx:uint64
&& cb.store_long_bool(delivered_lt, 64) // import_block_lt:uint64
&& insert_out_msg(cb.finalize(), out_queue_key.bits() + 96);
} else {
return cb.store_long_bool(12, 4) // msg_export_deq$1100
&& cb.store_ref_bool(msg_envelope) // out_msg:^MsgEnvelope
&& cb.store_long_bool(delivered_lt, 63) // import_block_lt:uint63
&& insert_out_msg(cb.finalize());
}
}
bool Collator::out_msg_queue_cleanup() {
@ -2783,7 +2794,7 @@ bool Collator::insert_in_msg(Ref<vm::Cell> in_msg) {
// inserts an OutMsg into OutMsgDescr
bool Collator::insert_out_msg(Ref<vm::Cell> out_msg) {
if (verbosity > 2) {
fprintf(stderr, "OutMsg being inserted into OutMsgDescr: ");
std::cerr << "OutMsg being inserted into OutMsgDescr: ";
block::gen::t_OutMsg.print_ref(std::cerr, out_msg);
}
auto cs = load_cell_slice(out_msg);
@ -2800,10 +2811,14 @@ bool Collator::insert_out_msg(Ref<vm::Cell> out_msg) {
}
msg = cs2.prefetch_ref(); // use hash of (Message Any)
}
return insert_out_msg(std::move(out_msg), msg->get_hash().bits());
}
bool Collator::insert_out_msg(Ref<vm::Cell> out_msg, td::ConstBitPtr msg_hash) {
bool ok;
try {
ok = out_msg_dict->set(msg->get_hash().bits(), 256, cs, vm::Dictionary::SetMode::Add);
} catch (vm::VmError) {
ok = out_msg_dict->set(msg_hash, 256, load_cell_slice(std::move(out_msg)), vm::Dictionary::SetMode::Add);
} catch (vm::VmError&) {
ok = false;
}
if (!ok) {
@ -2814,6 +2829,7 @@ bool Collator::insert_out_msg(Ref<vm::Cell> out_msg) {
return block_limit_status_->add_cell(std::move(out_msg)) &&
((out_descr_cnt_ & 63) || block_limit_status_->add_cell(out_msg_dict->get_root_cell()));
}
// enqueues a new Message into OutMsgDescr and OutMsgQueue
bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_remaining, ton::LogicalTime enqueued_lt) {
// 0. unpack src_addr and dest_addr

View file

@ -62,7 +62,7 @@ class LiteQuery : public td::actor::Actor {
enum {
default_timeout_msec = 4500, // 4.5 seconds
max_transaction_count = 16, // fetch at most 16 transactions in one query
client_method_gas_limit = 100000 // gas limit for liteServer.runSmcMethod
client_method_gas_limit = 300000 // gas limit for liteServer.runSmcMethod
};
enum {
ls_version = 0x101,
@ -160,8 +160,8 @@ class LiteQuery : public td::actor::Actor {
const BlockIdExt& blkid);
bool make_state_root_proof(Ref<vm::Cell>& proof, Ref<vm::Cell> state_root, Ref<vm::Cell> block_root,
const BlockIdExt& blkid);
bool make_shard_info_proof(Ref<vm::Cell>& proof, Ref<block::McShardHash>& info, ShardIdFull shard, ShardIdFull& true_shard,
Ref<vm::Cell>& leaf, bool& found, bool exact = true);
bool make_shard_info_proof(Ref<vm::Cell>& proof, Ref<block::McShardHash>& info, ShardIdFull shard,
ShardIdFull& true_shard, Ref<vm::Cell>& leaf, bool& found, bool exact = true);
bool make_shard_info_proof(Ref<vm::Cell>& proof, Ref<block::McShardHash>& info, ShardIdFull shard, bool exact = true);
bool make_shard_info_proof(Ref<vm::Cell>& proof, Ref<block::McShardHash>& info, AccountIdPrefixFull prefix);
bool make_shard_info_proof(Ref<vm::Cell>& proof, BlockIdExt& blkid, AccountIdPrefixFull prefix);

View file

@ -2609,7 +2609,7 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id
}
auto q_msg_env = (old_value.not_null() ? old_value : new_value)->prefetch_ref();
int tag = (int)out_msg_cs->prefetch_ulong(3);
// mode for msg_export_{ext,new,imm,tr,deq_imm,???,deq,tr_req}
// mode for msg_export_{ext,new,imm,tr,deq_imm,???,deq/deq_short,tr_req}
static const int tag_mode[8] = {0, 2, 0, 2, 1, 0, 1, 3};
static const char* tag_str[8] = {"ext", "new", "imm", "tr", "deq_imm", "???", "deq", "tr_req"};
if (tag < 0 || tag >= 8 || !(tag_mode[tag] & mode)) {
@ -2617,18 +2617,33 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id
<< out_msg_id.to_hex(352) << " has invalid tag " << tag << "(" << tag_str[tag & 7]
<< ")");
}
auto msg_env = out_msg_cs->prefetch_ref();
if (msg_env.is_null()) {
return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) + " is invalid");
}
auto msg = vm::load_cell_slice(msg_env).prefetch_ref();
if (msg.is_null()) {
return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) + " is invalid");
}
if (msg->get_hash().as_bitslice() != out_msg_id + 96) {
return reject_query("OutMsgDescr for "s + (out_msg_id + 96).to_hex(256) +
" contains a message with different hash "s + msg->get_hash().bits().to_hex(256));
bool is_short = (tag == 6 && (out_msg_cs->prefetch_ulong(4) &
1)); // msg_export_deq_short does not contain true MsgEnvelope / Message
Ref<vm::Cell> msg_env, msg;
td::Bits256 msg_env_hash;
block::gen::OutMsg::Record_msg_export_deq_short deq_short;
if (!is_short) {
msg_env = out_msg_cs->prefetch_ref();
if (msg_env.is_null()) {
return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) + " is invalid (contains no MsgEnvelope)");
}
msg_env_hash = msg_env->get_hash().bits();
msg = vm::load_cell_slice(msg_env).prefetch_ref();
if (msg.is_null()) {
return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) + " is invalid (contains no message)");
}
if (msg->get_hash().as_bitslice() != out_msg_id + 96) {
return reject_query("OutMsgDescr for "s + (out_msg_id + 96).to_hex(256) +
" contains a message with different hash "s + msg->get_hash().bits().to_hex(256));
}
} else {
if (!tlb::csr_unpack(out_msg_cs, deq_short)) { // parsing msg_export_deq_short$1101 ...
return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) +
" is invalid (cannot unpack msg_export_deq_short)");
}
msg_env_hash = deq_short.msg_env_hash;
}
//
if (mode == 1) {
// dequeued message
if (tag == 7) {
@ -2665,13 +2680,13 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id
" is a msg_export_tr_req referring to a reimport InMsgDescr that contains a MsgEnvelope "
"distinct from that originally kept in the old queue");
}
} else if (msg_env->get_hash().as_bitslice() != q_msg_env->get_hash().bits()) {
} else if (msg_env_hash != q_msg_env->get_hash().bits()) {
return reject_query("OutMsgDescr corresponding to dequeued message with key "s + out_msg_id.to_hex(352) +
" contains a MsgEnvelope distinct from that originally kept in the old queue");
}
} else {
// enqueued message
if (msg_env->get_hash().as_bitslice() != q_msg_env->get_hash().bits()) {
if (msg_env_hash != q_msg_env->get_hash().bits()) {
return reject_query("OutMsgDescr corresponding to "s + m_str[mode] + "queued message with key "s +
out_msg_id.to_hex(352) +
" contains a MsgEnvelope distinct from that stored in the new queue");
@ -2680,14 +2695,24 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id
// in all cases above, we have to check that all 352-bit key is correct (including first 96 bits)
// otherwise we might not be able to correctly recover OutMsgQueue entries starting from OutMsgDescr later
// or we might have several OutMsgQueue entries with different 352-bit keys all having the same last 256 bits (with the message hash)
if (is_short) {
// check out_msg_id using fields next_workchain:int32 next_addr_pfx:uint64 of msg_export_deq_short$1101
if (out_msg_id.get_int(32) != deq_short.next_workchain ||
(out_msg_id + 32).get_uint(64) != deq_short.next_addr_pfx) {
return reject_query(
PSTRING() << "OutMsgQueue entry with key " << out_msg_id.to_hex(352)
<< " corresponds to msg_export_deq_short OutMsg entry with incorrect next hop parameters "
<< deq_short.next_workchain << "," << deq_short.next_addr_pfx);
}
}
td::BitArray<352> key;
if (!block::compute_out_msg_queue_key(q_msg_env, key)) {
return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) +
return reject_query("OutMsgQueue entry with key "s + out_msg_id.to_hex(352) +
" refers to a MsgEnvelope that cannot be unpacked");
}
if (key != out_msg_id) {
return reject_query("OutMsgDescr for "s + out_msg_id.to_hex(352) +
" contains a MsgEnvelope that should be stored under different key " + key.to_hex());
return reject_query("OutMsgQueue entry with key "s + out_msg_id.to_hex(352) +
" contains a MsgEnvelope that should have been stored under different key " + key.to_hex());
}
return true;
}
@ -2732,7 +2757,6 @@ bool ValidateQuery::update_min_enqueued_lt_hash(ton::LogicalTime lt, const ton::
// check that the enveloped message (MsgEnvelope) was present in the output queue of a neighbor, and that it has not been processed before
bool ValidateQuery::check_imported_message(Ref<vm::Cell> msg_env) {
// TODO
block::tlb::MsgEnvelope::Record_std env;
block::gen::CommonMsgInfo::Record_int_msg_info info;
ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix;
@ -3231,6 +3255,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref<vm::CellSlice> out_ms
Ref<vm::CellSlice> src, dest;
Ref<vm::Cell> transaction;
Ref<vm::Cell> msg, msg_env, tr_msg_env, reimport;
td::Bits256 msg_env_hash;
// msg_envelope#4 cur_addr:IntermediateAddress next_addr:IntermediateAddress fwd_fee_remaining:Grams msg:^(Message Any) = MsgEnvelope;
block::tlb::MsgEnvelope::Record_std env;
// int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool
@ -3243,6 +3268,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref<vm::CellSlice> out_ms
ton::LogicalTime import_lt = ~0ULL;
unsigned long long created_lt = 0;
int mode = 0, in_tag = -2;
bool is_short = false;
// initial checks and unpack
switch (tag) {
case block::gen::OutMsg::msg_export_ext: {
@ -3317,6 +3343,18 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref<vm::CellSlice> out_ms
// ...
break;
}
case block::gen::OutMsg::msg_export_deq_short: {
block::gen::OutMsg::Record_msg_export_deq_short out;
CHECK(tlb::csr_unpack(out_msg, out));
msg_env_hash = out.msg_env_hash;
next_prefix.workchain = out.next_workchain;
next_prefix.account_id_prefix = out.next_addr_pfx;
import_lt = out.import_block_lt;
is_short = true;
mode = 1; // removed from OutMsgQueue
// ...
break;
}
case block::gen::OutMsg::msg_export_tr_req: {
block::gen::OutMsg::Record_msg_export_tr_req out;
CHECK(tlb::csr_unpack(out_msg, out) && tlb::unpack_cell(out.out_msg, env));
@ -3343,15 +3381,22 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref<vm::CellSlice> out_ms
return reject_query(PSTRING() << "OutMsg with key (message hash) " << key.to_hex(256) << " has an unknown tag "
<< tag);
}
// common checks for all (non-external) inbound messages
CHECK(msg.not_null());
if (msg->get_hash().as_bitslice() != key) {
return reject_query("OutMsg with key "s + key.to_hex(256) + " refers to a message with different hash " +
msg->get_hash().to_hex());
if (msg_env.not_null()) {
msg_env_hash = msg_env->get_hash().bits();
}
if (tag != block::gen::OutMsg::msg_export_ext) {
// common checks for all (non-external) outbound messages
if (!is_short) {
CHECK(msg.not_null());
if (msg->get_hash().as_bitslice() != key) {
return reject_query("OutMsg with key "s + key.to_hex(256) + " refers to a message with different hash " +
msg->get_hash().to_hex());
}
}
if (is_short) {
// nothing to check here for msg_export_deq_short ?
} else if (tag != block::gen::OutMsg::msg_export_ext) {
// unpack int_msg_info$0 ... = CommonMsgInfo, especially message addresses
if (!tlb::unpack_cell_inexact(msg, info)) {
return reject_query("OutMsg with key "s + key.to_hex(256) +
@ -3447,7 +3492,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref<vm::CellSlice> out_ms
td::BitArray<32 + 64 + 256> q_key;
q_key.bits().store_int(next_prefix.workchain, 32);
(q_key.bits() + 32).store_int(next_prefix.account_id_prefix, 64);
(q_key.bits() + 96).copy_from(env.msg->get_hash().bits(), 256);
(q_key.bits() + 96).copy_from(key, 256);
auto q_entry = ns_.out_msg_queue_->lookup(q_key);
auto old_q_entry = ps_.out_msg_queue_->lookup(q_key);
if (old_q_entry.not_null() && q_entry.not_null()) {
@ -3471,7 +3516,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref<vm::CellSlice> out_ms
return reject_query("OutMsg with key "s + key.to_hex(256) +
" was expected to create OutMsgQueue entry with key " + q_key.to_hex() + " but it did not");
}
if (q_entry->prefetch_ref()->get_hash() != msg_env->get_hash()) {
if (msg_env_hash != q_entry->prefetch_ref()->get_hash().bits()) {
return reject_query("OutMsg with key "s + key.to_hex(256) + " has created OutMsgQueue entry with key " +
q_key.to_hex() + " containing a different MsgEnvelope");
}
@ -3482,7 +3527,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref<vm::CellSlice> out_ms
" was expected to remove OutMsgQueue entry with key " + q_key.to_hex() +
" but it did not exist in the old queue");
}
if (old_q_entry->prefetch_ref()->get_hash() != msg_env->get_hash()) {
if (msg_env_hash != old_q_entry->prefetch_ref()->get_hash().bits()) {
return reject_query("OutMsg with key "s + key.to_hex(256) + " has dequeued OutMsgQueue entry with key " +
q_key.to_hex() + " containing a different MsgEnvelope");
}
@ -3599,7 +3644,8 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref<vm::CellSlice> out_ms
// ...
break;
}
case block::gen::OutMsg::msg_export_deq: {
case block::gen::OutMsg::msg_export_deq:
case block::gen::OutMsg::msg_export_deq_short: {
// check that the message has been indeed processed by a neighbor
CHECK(old_q_entry.not_null());
block::EnqueuedMsgDescr enq_msg_descr;
@ -3841,14 +3887,30 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref<vm::CellSlice> enq_msg,
" already processed by this shard, but there is no ext_message_deq OutMsg record for this "
"message in this block");
}
block::gen::OutMsg::Record_msg_export_deq deq;
if (!tlb::csr_unpack(std::move(out_entry), deq)) {
return reject_query("cannot unpack msg_export_deq OutMsg record for already processed EnqueuedMsg with key "s +
key.to_hex(352) + " of old outbound queue");
}
if (deq.out_msg->get_hash() != enq.msg_env_->get_hash()) {
return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s +
key.to_hex(352) + " of old outbound queue contains a different MsgEnvelope");
int tag = block::gen::t_OutMsg.get_tag(*out_entry);
if (tag == block::gen::OutMsg::msg_export_deq_short) {
block::gen::OutMsg::Record_msg_export_deq_short deq;
if (!tlb::csr_unpack(std::move(out_entry), deq)) {
return reject_query(
"cannot unpack msg_export_deq_short OutMsg record for already processed EnqueuedMsg with key "s +
key.to_hex(352) + " of old outbound queue");
}
if (deq.msg_env_hash != enq.msg_env_->get_hash().bits()) {
return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s +
key.to_hex(352) + " of old outbound queue refers to MsgEnvelope with different hash " +
deq.msg_env_hash.to_hex());
}
} else {
block::gen::OutMsg::Record_msg_export_deq deq;
if (!tlb::csr_unpack(std::move(out_entry), deq)) {
return reject_query(
"cannot unpack msg_export_deq OutMsg record for already processed EnqueuedMsg with key "s +
key.to_hex(352) + " of old outbound queue");
}
if (deq.out_msg->get_hash() != enq.msg_env_->get_hash()) {
return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s +
key.to_hex(352) + " of old outbound queue contains a different MsgEnvelope");
}
}
}
// next check is incorrect after a merge, when ns_.processed_upto has > 1 entries