From 53fd88b67a04ed03698da1e693cd66ba9ffccabc Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 26 Apr 2023 09:15:16 +0300 Subject: [PATCH 1/4] Add timeout for out_msg_queue_cleanup (#679) Co-authored-by: SpyCheese --- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index dfe844a3..7b136953 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -73,7 +73,7 @@ class Collator final : public td::actor::Actor { Ref validator_set_; td::actor::ActorId manager; td::Timestamp timeout; - td::Timestamp soft_timeout_, medium_timeout_; + td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; ton::BlockSeqno last_block_seqno{0}; ton::BlockSeqno prev_mc_block_seqno{0}; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 83bc710c..6debf2de 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -67,6 +67,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockId , validator_set_(std::move(validator_set)) , manager(manager) , timeout(timeout) + , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 5.0)) , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) @@ -1814,6 +1815,11 @@ bool Collator::out_msg_queue_cleanup() { auto res = out_msg_queue_->filter([&](vm::CellSlice& cs, td::ConstBitPtr key, int n) -> int { assert(n == 352); // LOG(DEBUG) << "key is " << key.to_hex(n); + if (queue_cleanup_timeout_.is_in_past(td::Timestamp::now())) { + LOG(WARNING) << "cleaning up outbound queue takes too long, ending"; + outq_cleanup_partial_ = true; + return (1 << 30) + 1; // retain all remaining outbound queue entries including this one without processing + } if (block_full_) { LOG(WARNING) << "BLOCK FULL while cleaning up outbound queue, cleanup completed only partially"; outq_cleanup_partial_ = true; From f44dce93b632e9c1cd28baa7da079895db2c7c7f Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 26 Apr 2023 21:32:26 +0300 Subject: [PATCH 2/4] Make stricter cleanup timelimit --- validator/impl/collator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6debf2de..6935f83c 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -67,7 +67,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockId , validator_set_(std::move(validator_set)) , manager(manager) , timeout(timeout) - , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 5.0)) + , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 9.0)) , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) From 6786f1d5b4bda03e044f57fb210944f7ee6147e9 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 27 Apr 2023 14:43:37 +0300 Subject: [PATCH 3/4] Comment out excessive checks of outbound messages queue --- validator/impl/collator.cpp | 1 + validator/impl/validate-query.cpp | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6935f83c..1c3cafe6 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -67,6 +67,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockId , validator_set_(std::move(validator_set)) , manager(manager) , timeout(timeout) + // default timeout is 10 seconds, declared in validator/validator-group.cpp:generate_block_candidate:run_collate_query , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 9.0)) , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 7d8104e0..0e1b1199 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -5540,9 +5540,10 @@ bool ValidateQuery::try_validate() { if (!check_in_queue()) { return reject_query("cannot check inbound message queues"); } - if (!check_delivered_dequeued()) { + // Excessive check: validity of message in queue is checked elsewhere + /*if (!check_delivered_dequeued()) { return reject_query("cannot check delivery status of all outbound messages"); - } + }*/ if (!check_transactions()) { return reject_query("invalid collection of account transactions in ShardAccountBlocks"); } From 67e64e5d157d1d78311817ec37b866fe4116e64d Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 28 Apr 2023 16:42:11 +0300 Subject: [PATCH 4/4] Revert too strictening of queue_cleanup_timeout --- validator/impl/collator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 1c3cafe6..4e68b07c 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -68,7 +68,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockId , manager(manager) , timeout(timeout) // default timeout is 10 seconds, declared in validator/validator-group.cpp:generate_block_candidate:run_collate_query - , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 9.0)) + , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 5.0)) , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise))