diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 4393c6e2..86e7e5b8 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -151,6 +151,9 @@ message$_ {X:Type} info:CommonMsgInfo message$_ {X:Type} info:CommonMsgInfoRelaxed init:(Maybe (Either StateInit ^StateInit)) body:(Either X ^X) = MessageRelaxed X; + +_ (Message Any) = MessageAny; + // interm_addr_regular$0 use_dest_bits:(#<= 96) = IntermediateAddress; diff --git a/crypto/smartcont/highload-wallet-v2-one.fif b/crypto/smartcont/highload-wallet-v2-one.fif new file mode 100644 index 00000000..bd7fa5aa --- /dev/null +++ b/crypto/smartcont/highload-wallet-v2-one.fif @@ -0,0 +1,112 @@ +#!/usr/bin/fift -s +"TonUtil.fif" include +"GetOpt.fif" include + +{ show-options-help 1 halt } : usage + +"" =: comment // comment for simple transfers +true =: allow-bounce +false =: force-bounce +3 =: send-mode // mode for SENDRAWMSG: +1 - sender pays fees, +2 - ignore errors +60 =: timeout // external message expires in 60 seconds +variable extra-currencies +{ extra-currencies @ cc+ extra-currencies ! } : extra-cc+! + +begin-options + " [-x *] [-n|-b] [-t] [-B ] [-C ] []" +cr +tab + +"Creates one request to highload wallet created by new-highload-wallet-v2.fif, with private key loaded from file .pk " + +"and address from .addr, and saves it into .boc ('wallet-query.boc' by default)" + disable-digit-options generic-help-setopt + "n" "--no-bounce" { false =: allow-bounce } short-long-option + "Clears bounce flag" option-help + "b" "--force-bounce" { true =: force-bounce } short-long-option + "Forces bounce flag" option-help + "x" "--extra" { $>xcc extra-cc+! } short-long-option-arg + "Indicates the amount of extra currencies to be transfered" option-help + "t" "--timeout" { parse-int =: timeout } short-long-option-arg + "Sets expiration timeout in seconds (" timeout (.) $+ +" by default)" option-help + "B" "--body" { =: body-boc-file } short-long-option-arg + "Sets the payload of the transfer message" option-help + "C" "--comment" { =: comment } short-long-option-arg + "Sets the comment to be sent in the transfer message" option-help + "m" "--mode" { parse-int =: send-mode } short-long-option-arg + "Sets transfer mode (0..255) for SENDRAWMSG (" send-mode (.) $+ +" by default)" + option-help + "h" "--help" { usage } short-long-option + "Shows a help message" option-help +parse-options + +$# dup 4 < swap 5 > or ' usage if +5 :$1..n + +true constant bounce +$1 =: file-base +$2 bounce parse-load-address force-bounce or allow-bounce and =: bounce 2=: dest_addr +$3 parse-int =: subwallet-id +$4 $>cc extra-cc+! extra-currencies @ 2=: amount +$5 "wallet-query" replace-if-null =: savefile +{ subwallet-id (.) $+ } : +subwallet + +file-base +subwallet +".addr" load-address +2dup 2constant wallet_addr +."Source wallet address = " 2dup .addr cr 6 .Addr cr +file-base +".pk" load-keypair nip constant wallet_pk + +def? body-boc-file { @' body-boc-file file>B B>boc } { comment simple-transfer-body } cond +constant body-cell + +."Transferring " amount .GR+cc ."to account " +dest_addr 2dup bounce 7 + .Addr ." = " .addr +."subwallet-id=0x" subwallet-id x. +."timeout=" timeout . ."bounce=" bounce . cr +."Body of transfer message is " body-cell = abort"more than 254 orders" + orders @ 16 udict!+ not abort"cannot add order to dictionary" + orders ! order# 1+! +} : add-order +// b body -- b' +{ tuck +} : create-int-msg +// ng wc addr bnc -- +{ ."Transferring " 3 roll .GR ."to account " + -rot 2dup 4 pick 7 + .Addr ." = " .addr ." bounce=" . cr +} : .transfer +// addr$ ng -- c +{ swap parse-smc-addr force-bounce or allow-bounce and // ng wc addr bnc + 2over 2over .transfer + create-int-msg +} : create-simple-transfer +// c m -- c' +{ } : create-order + +// addr$ ng -- +{ create-simple-transfer send-mode create-order add-order } : send +{ bl word bl word $>GR send } : SEND + +// create internal message + +send-mode create-order add-order + +// create external message +now timeout + 32 << hashu 32 1<<1- and + =: query_id + +dup ."signing message: " +dup ."resulting external message: " B dup Bx. cr +."Query_id is " query_id dup . ."= 0x" X. cr +savefile +".boc" tuck B>file +."(Saved to file " type .")" cr diff --git a/crypto/smartcont/wallet.fif b/crypto/smartcont/wallet.fif index 4c0cd374..5fc9141d 100755 --- a/crypto/smartcont/wallet.fif +++ b/crypto/smartcont/wallet.fif @@ -26,6 +26,8 @@ begin-options "Sets the payload of the transfer message" option-help "C" "--comment" { =: comment } short-long-option-arg "Sets the comment to be sent in the transfer message" option-help + "I" "--with-init" { =: init-file } short-long-option-arg + "Indicates filename with BoC containing StateInit for internal message" option-help "m" "--mode" { parse-int =: send-mode } short-long-option-arg "Sets transfer mode (0..255) for SENDRAWMSG (" send-mode (.) $+ +" by default)" option-help @@ -52,13 +54,17 @@ file-base +".pk" load-keypair nip constant wallet_pk def? body-boc-file { @' body-boc-file file>B B>boc } { comment simple-transfer-body } cond constant body-cell +def? init-file { @' init-file file>B B>boc diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index fc94a2dc..ac32b38f 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -1670,13 +1670,18 @@ Ref DictionaryFixed::extract_prefix_subdict_root(td::ConstBitPtr prefi } std::pair, int> DictionaryFixed::dict_filter(Ref dict, td::BitPtr key, int n, - const DictionaryFixed::filter_func_t& check_leaf) const { + const DictionaryFixed::filter_func_t& check_leaf, + int& skip_rest) const { // std::cerr << "dictionary filter for " << n << "-bit key = " << (key + n - key_bits).to_hex(key_bits - n) // << std::endl; if (dict.is_null()) { // empty dictionary, return unchanged return {{}, 0}; } + if (skip_rest >= 0) { + // either drop subtree completely (if skip_rest>0), or retain it completely (if skip_rest=0) + return {{}, skip_rest}; + } LabelParser label{std::move(dict), n, label_mode()}; assert(label.l_bits >= 0 && label.l_bits <= n); label.extract_label_to(key); @@ -1684,6 +1689,11 @@ std::pair, int> DictionaryFixed::dict_filter(Ref dict, td::BitPt if (label.l_bits == n) { // leaf int res = check_leaf(label.remainder.write(), key - key_bits, key_bits); + if (res >= (1 << 30)) { + // skip all, or retain all + res &= (1 << 30) - 1; + skip_rest = (res ? 0 : (1 << 30)); + } return {{}, res < 0 ? res : !res}; } // fork, process left and right subtrees @@ -1691,19 +1701,20 @@ std::pair, int> DictionaryFixed::dict_filter(Ref dict, td::BitPt key[-1] = false; int delta = label.l_bits + 1; n -= delta; - auto left_res = dict_filter(label.remainder->prefetch_ref(0), key, n, check_leaf); + auto left_res = dict_filter(label.remainder->prefetch_ref(0), key, n, check_leaf, skip_rest); if (left_res.second < 0) { return left_res; } key[-1] = true; - auto right_res = dict_filter(label.remainder->prefetch_ref(1), key, n, check_leaf); + auto right_res = dict_filter(label.remainder->prefetch_ref(1), key, n, check_leaf, skip_rest); if ((left_res.second | right_res.second) <= 0) { // error in right, or both left and right unchanged return right_res; } auto left = left_res.second ? std::move(left_res.first) : label.remainder->prefetch_ref(0); auto right = right_res.second ? std::move(right_res.first) : label.remainder->prefetch_ref(1); - auto changes = left_res.second + right_res.second; + // 2^30 is effectively infinity, meaning that we dropped whole branches with unknown # of nodes + auto changes = ((left_res.second | right_res.second) & (1 << 30)) ? (1 << 30) : left_res.second + right_res.second; label.clear(); if (left.is_null()) { if (right.is_null()) { @@ -1735,8 +1746,9 @@ std::pair, int> DictionaryFixed::dict_filter(Ref dict, td::BitPt int DictionaryFixed::filter(DictionaryFixed::filter_func_t check_leaf) { force_validate(); + int skip_rest = -1; unsigned char buffer[DictionaryFixed::max_key_bytes]; - auto res = dict_filter(get_root_cell(), td::BitPtr{buffer}, key_bits, check_leaf); + auto res = dict_filter(get_root_cell(), td::BitPtr{buffer}, key_bits, check_leaf, skip_rest); if (res.second > 0) { // std::cerr << "after filter (" << res.second << " changes): new augmented dictionary root is:\n"; // vm::load_cell_slice(res.first).print_rec(std::cerr); diff --git a/crypto/vm/dict.h b/crypto/vm/dict.h index 761c4073..978f4d53 100644 --- a/crypto/vm/dict.h +++ b/crypto/vm/dict.h @@ -293,7 +293,8 @@ class DictionaryFixed : public DictionaryBase { bool remove_prefix = false) const; bool dict_check_for_each(Ref dict, td::BitPtr key_buffer, int n, int total_key_len, const foreach_func_t& foreach_func, bool invert_first = false) const; - std::pair, int> dict_filter(Ref dict, td::BitPtr key, int n, const filter_func_t& check_leaf) const; + std::pair, int> dict_filter(Ref dict, td::BitPtr key, int n, const filter_func_t& check_leaf, + int& skip_rest) const; Ref dict_combine_with(Ref dict1, Ref dict2, td::BitPtr key_buffer, int n, int total_key_len, const combine_func_t& combine_func, int mode = 0, int skip1 = 0, int skip2 = 0) const; bool dict_scan_diff(Ref dict1, Ref dict2, td::BitPtr key_buffer, int n, int total_key_len, diff --git a/doc/ConfigParam-HOWTO b/doc/ConfigParam-HOWTO index 6dd6609b..8432657a 100644 --- a/doc/ConfigParam-HOWTO +++ b/doc/ConfigParam-HOWTO @@ -215,7 +215,7 @@ We see that the list of all active configuration proposals consists of exactly o [6465...6321 [1586779536 0 [8 C{FDCD...} -1] 1124...2998 () 8646...209 3 0 0]] -Here the first number 6465..6321 is the unique identifier of the configuration proposal, equal to its 256-bit hash. The second component of this pair is a Tuple describing the status of this configuration proposal. The first component of this Tuple is the expiration Unixtime of the configuration proposal (1586779546). The second component (0) is the criticality flag. Next comes the configuration proposal proper, described by triple [8 C{FDCD...} -1], where 8 is the index of the configuration parameter to be modified, C{FDCD...} is the cell with the new value (represented by the hash of this cell), and -1 is the optional hash of the old value of this parameter (-1 means that this hash has not been specified). Next we see a large number 1124...2998 representing the identifier of the current validator set, then an empty list () representing the set of all currently active validators that have voted for this proposal so far, then *weight_remaining* equal to 8646...209 - a number that is positive if the proposal has not yet collected enough validator votes in this round, and negative otherwise. Then we see three numbers 3 0 0. These numbers are *rounds_remaining* (this proposal will survive at most three rounds, i.e., changes of the current validator set), *wins* (the count of rounds where the proposal collected votes of more than 3/4 of all validators by weight) and *losses* (the count of rounds where the proposal failed to collect 3/4 of all validator votes). +Here the first number 6465..6321 is the unique identifier of the configuration proposal, equal to its 256-bit hash. The second component of this pair is a Tuple describing the status of this configuration proposal. The first component of this Tuple is the expiration Unixtime of the configuration proposal (1586779536). The second component (0) is the criticality flag. Next comes the configuration proposal proper, described by triple [8 C{FDCD...} -1], where 8 is the index of the configuration parameter to be modified, C{FDCD...} is the cell with the new value (represented by the hash of this cell), and -1 is the optional hash of the old value of this parameter (-1 means that this hash has not been specified). Next we see a large number 1124...2998 representing the identifier of the current validator set, then an empty list () representing the set of all currently active validators that have voted for this proposal so far, then *weight_remaining* equal to 8646...209 - a number that is positive if the proposal has not yet collected enough validator votes in this round, and negative otherwise. Then we see three numbers 3 0 0. These numbers are *rounds_remaining* (this proposal will survive at most three rounds, i.e., changes of the current validator set), *wins* (the count of rounds where the proposal collected votes of more than 3/4 of all validators by weight) and *losses* (the count of rounds where the proposal failed to collect 3/4 of all validator votes). We can inspect the proposed value for configuration parameter #8 by asking the lite-client to expand cell C{FDCD...} using its hash FDCD... or a sufficiently long prefix of this hash to uniquely identify the cell in question: diff --git a/validator-session/validator-session-description.cpp b/validator-session/validator-session-description.cpp index f6e6f582..c1a45dd6 100644 --- a/validator-session/validator-session-description.cpp +++ b/validator-session/validator-session-description.cpp @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - Copyright 2017-2019 Telegram Systems LLP + Copyright 2017-2020 Telegram Systems LLP */ #include "validator-session.hpp" #include "td/utils/Random.h" @@ -52,10 +52,10 @@ ValidatorSessionDescriptionImpl::ValidatorSessionDescriptionImpl(ValidatorSessio self_idx_ = it->second; pdata_temp_ptr_ = 0; - pdata_temp_size_ = 1 << 30; + pdata_temp_size_ = 1 << 27; pdata_temp_ = new td::uint8[pdata_temp_size_]; - pdata_perm_size_ = 1ull << 30; + pdata_perm_size_ = 1ull << 27; pdata_perm_ptr_ = 0; for (auto &el : cache_) { diff --git a/validator-session/validator-session-description.hpp b/validator-session/validator-session-description.hpp index d7b430bd..d65369ac 100644 --- a/validator-session/validator-session-description.hpp +++ b/validator-session/validator-session-description.hpp @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - Copyright 2017-2019 Telegram Systems LLP + Copyright 2017-2020 Telegram Systems LLP */ #pragma once diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 0e706c35..dad2fe8b 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -59,6 +59,7 @@ class Collator final : public td::actor::Actor { bool preinit_complete{false}; bool is_key_block_{false}; bool block_full_{false}; + bool outq_cleanup_partial_{false}; bool inbound_queues_empty_{false}; bool libraries_changed_{false}; bool prev_key_block_exists_{false}; @@ -144,6 +145,8 @@ class Collator final : public td::actor::Actor { bool ihr_enabled_{false}; bool create_stats_enabled_{false}; bool report_version_{false}; + bool skip_topmsgdescr_{false}; + bool skip_extmsg_{false}; td::uint64 overload_history_{0}, underload_history_{0}; td::uint64 block_size_estimate_{}; Ref wc_info_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 06d81897..f4f2d869 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -34,6 +34,7 @@ #include "validator-set.hpp" #include "top-shard-descr.hpp" #include +#include "td/utils/Random.h" namespace ton { @@ -1208,6 +1209,9 @@ bool Collator::import_new_shard_top_blocks() { if (shard_block_descr_.empty()) { return true; } + if (skip_topmsgdescr_) { + return true; + } auto lt_limit = config_->lt + config_->get_max_lt_growth(); std::sort(shard_block_descr_.begin(), shard_block_descr_.end(), cmp_shard_block_descr_ref); int tb_act = 0; @@ -1442,6 +1446,37 @@ bool Collator::init_utime() { "error initializing unix time for the new block: failed to observe end of fsm_split time interval for this " "shard"); } + // check whether masterchain catchain rotation is overdue + auto ccvc = config_->get_catchain_validators_config(); + unsigned lifetime = ccvc.mc_cc_lifetime; + if (is_masterchain() && now_ / lifetime > prev_now_ / lifetime && now_ > (prev_now_ / lifetime + 1) * lifetime + 20) { + auto overdue = now_ - (prev_now_ / lifetime + 1) * lifetime; + // masterchain catchain rotation overdue, skip topsharddescr with some probability + skip_topmsgdescr_ = (td::Random::fast(0, 1023) < 256); // probability 1/4 + skip_extmsg_ = (td::Random::fast(0, 1023) < 256); // skip ext msg probability 1/4 + if (skip_topmsgdescr_) { + LOG(WARNING) + << "randomly skipping import of new shard data because of overdue masterchain catchain rotation (overdue by " + << overdue << " seconds)"; + } + if (skip_extmsg_) { + LOG(WARNING) + << "randomly skipping external message import because of overdue masterchain catchain rotation (overdue by " + << overdue << " seconds)"; + } + } else if (is_masterchain() && now_ > prev_now_ + 60) { + auto interval = now_ - prev_now_; + skip_topmsgdescr_ = (td::Random::fast(0, 1023) < 128); // probability 1/8 + skip_extmsg_ = (td::Random::fast(0, 1023) < 128); // skip ext msg probability 1/8 + if (skip_topmsgdescr_) { + LOG(WARNING) << "randomly skipping import of new shard data because of overdue masterchain block (last block was " + << interval << " seconds ago)"; + } + if (skip_extmsg_) { + LOG(WARNING) << "randomly skipping external message import because of overdue masterchain block (last block was " + << interval << " seconds ago)"; + } + } return true; } @@ -1742,7 +1777,7 @@ bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime deli } bool Collator::out_msg_queue_cleanup() { - LOG(DEBUG) << "in out_msg_queue_cleanup()"; + LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors"; if (verbosity >= 2) { auto rt = out_msg_queue_->get_root(); std::cerr << "old out_msg_queue is "; @@ -1759,6 +1794,11 @@ bool Collator::out_msg_queue_cleanup() { auto res = out_msg_queue_->filter([&](vm::CellSlice& cs, td::ConstBitPtr key, int n) -> int { assert(n == 352); // LOG(DEBUG) << "key is " << key.to_hex(n); + if (block_full_) { + LOG(WARNING) << "BLOCK FULL while cleaning up outbound queue, cleanup completed only partially"; + outq_cleanup_partial_ = true; + return (1 << 30) + 1; // retain all remaining outbound queue entries including this one without processing + } block::EnqueuedMsgDescr enq_msg_descr; unsigned long long created_lt; if (!(cs.fetch_ulong_bool(64, created_lt) // augmentation @@ -1789,6 +1829,10 @@ bool Collator::out_msg_queue_cleanup() { << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record"); return -1; } + register_out_msg_queue_op(); + if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { + block_full_ = true; + } } return !delivered; }); @@ -2631,6 +2675,10 @@ bool Collator::process_inbound_internal_messages() { } bool Collator::process_inbound_external_messages() { + if (skip_extmsg_) { + LOG(INFO) << "skipping processing of inbound external messages"; + return true; + } bool full = !block_limit_status_->fits(block::ParamLimits::cl_soft); for (auto& ext_msg_pair : ext_msg_list_) { if (full) { diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index d512063f..0912de88 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -204,7 +204,7 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { } td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [Self = actor_id(this), mode](td::Result, BlockIdExt>> res) { + [ Self = actor_id(this), mode ](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -242,7 +242,7 @@ void LiteQuery::perform_getBlock(BlockIdExt blkid) { return; } td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { + [ Self = actor_id(this), blkid ](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -268,7 +268,7 @@ void LiteQuery::perform_getBlockHeader(BlockIdExt blkid, int mode) { return; } td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self = actor_id(this), blkid, mode](td::Result> res) { + [ Self = actor_id(this), blkid, mode ](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -383,7 +383,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { } if (blkid.id.seqno) { td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { + [ Self = actor_id(this), blkid ](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -393,7 +393,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { }); } else { td::actor::send_closure_later(manager_, &ValidatorManager::get_zero_state, blkid, - [Self = actor_id(this), blkid](td::Result res) { + [ Self = actor_id(this), blkid ](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -452,7 +452,7 @@ bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { + [ Self = actor_id(this), blkid ](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); @@ -478,7 +478,7 @@ bool LiteQuery::request_mc_proof(BlockIdExt blkid, int mode) { ++pending_; td::actor::send_closure( manager_, &ValidatorManager::get_key_block_proof, blkid, - [Self = actor_id(this), manager = manager_, blkid, mode](td::Result R) { + [ Self = actor_id(this), manager = manager_, blkid, mode ](td::Result R) { if (R.is_ok()) { auto proof = create_proof(blkid, R.move_as_ok()); proof.ensure(); @@ -510,7 +510,7 @@ bool LiteQuery::request_mc_block_state(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { + [ Self = actor_id(this), blkid ](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); @@ -541,7 +541,7 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { + [ Self = actor_id(this), blkid ](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); @@ -563,7 +563,7 @@ bool LiteQuery::request_block_data(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { + [ Self = actor_id(this), blkid ](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); @@ -586,7 +586,7 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) { if (blkid.is_masterchain()) { td::actor::send_closure( manager_, &ValidatorManager::get_key_block_proof_link, blkid, - [Self = actor_id(this), manager = manager_, blkid](td::Result R) { + [ Self = actor_id(this), manager = manager_, blkid ](td::Result R) { if (R.is_ok()) { auto proof = create_proof(blkid, R.move_as_ok()); proof.ensure(); @@ -608,7 +608,7 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) { } else { td::actor::send_closure_later( manager_, &ValidatorManager::get_block_proof_link_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { + [ Self = actor_id(this), blkid ](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : ")); @@ -634,7 +634,7 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_zero_state, blkid, - [Self = actor_id(this), blkid](td::Result res) { + [ Self = actor_id(this), blkid ](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : ")); @@ -679,7 +679,7 @@ void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, LOG(INFO) << "sending a get_top_masterchain_state_block query to manager"; td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [Self = actor_id(this)](td::Result, BlockIdExt>> res) -> void { + [Self = actor_id(this)](td::Result, BlockIdExt>> res)->void { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -890,7 +890,7 @@ bool LiteQuery::make_state_root_proof(Ref& proof, Ref state_ return true; } -bool LiteQuery::make_shard_info_proof(Ref& proof, vm::CellSlice& cs, ShardIdFull shard, +bool LiteQuery::make_shard_info_proof(Ref& proof, Ref& info, ShardIdFull shard, ShardIdFull& true_shard, Ref& leaf, bool& found, bool exact) { vm::MerkleProofBuilder pb{mc_state_->root_cell()}; block::gen::ShardStateUnsplit::Record sstate; @@ -901,7 +901,16 @@ bool LiteQuery::make_shard_info_proof(Ref& proof, vm::CellSlice& cs, S if (!shards_dict) { return fatal_error("cannot extract ShardHashes from last mc state"); } + vm::CellSlice cs; found = block::ShardConfig::get_shard_hash_raw_from(*shards_dict, cs, shard, true_shard, exact, &leaf); + if (found) { + info = block::McShardHash::unpack(cs, true_shard); + if (info.is_null()) { + return fatal_error("cannot unpack a leaf entry from ShardHashes"); + } + } else { + info.clear(); + } if (!pb.extract_proof_to(proof)) { return fatal_error("unknown error creating Merkle proof"); } @@ -911,21 +920,9 @@ bool LiteQuery::make_shard_info_proof(Ref& proof, vm::CellSlice& cs, S bool LiteQuery::make_shard_info_proof(Ref& proof, Ref& info, ShardIdFull shard, bool exact) { Ref leaf; - vm::CellSlice cs; ShardIdFull true_shard; bool found; - if (!make_shard_info_proof(proof, cs, shard, true_shard, leaf, found, exact)) { - return false; - } - if (found) { - info = block::McShardHash::unpack(cs, true_shard); - if (info.is_null()) { - return fatal_error("cannot unpack a leaf entry from ShardHashes"); - } - } else { - info.clear(); - } - return true; + return make_shard_info_proof(proof, info, shard, true_shard, leaf, found, exact); } bool LiteQuery::make_shard_info_proof(Ref& proof, Ref& info, AccountIdPrefixFull prefix) { @@ -1271,14 +1268,14 @@ void LiteQuery::continue_getTransactions(unsigned remaining, bool exact) { << " " << trans_lt_; td::actor::send_closure_later( manager_, &ValidatorManager::get_block_by_lt_from_db, ton::extract_addr_prefix(acc_workchain_, acc_addr_), - trans_lt_, [Self = actor_id(this), remaining, manager = manager_](td::Result res) { + trans_lt_, [ Self = actor_id(this), remaining, manager = manager_ ](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, res.move_as_error(), ton::BlockIdExt{}); } else { auto handle = res.move_as_ok(); LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, - [Self, blkid = handle->id(), remaining](td::Result> res) { + [ Self, blkid = handle->id(), remaining ](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, res.move_as_error(), blkid); @@ -1345,7 +1342,7 @@ void LiteQuery::perform_getShardInfo(BlockIdExt blkid, ShardIdFull shard, bool e void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector param_list) { LOG(INFO) << "started a getConfigParams(" << blkid.to_str() << ", " << mode << ", ) liteserver query"; - set_continuation([this, mode, param_list = std::move(param_list)]() mutable { + set_continuation([ this, mode, param_list = std::move(param_list) ]() mutable { continue_getConfigParams(mode, std::move(param_list)); }); request_mc_block_data_state(blkid); @@ -1412,7 +1409,8 @@ void LiteQuery::continue_getShardInfo(ShardIdFull shard, bool exact) { vm::CellSlice cs; ShardIdFull true_shard; bool found; - if (!make_shard_info_proof(proof2, cs, shard, true_shard, leaf, found, exact)) { + Ref shard_info; + if (!make_shard_info_proof(proof2, shard_info, shard, true_shard, leaf, found, exact)) { return; } auto proof = vm::std_boc_serialize_multi({std::move(proof1), std::move(proof2)}); @@ -1423,7 +1421,6 @@ void LiteQuery::continue_getShardInfo(ShardIdFull shard, bool exact) { BlockIdExt true_id; td::BufferSlice data; if (found) { - auto shard_info = block::McShardHash::unpack(cs, true_shard); if (shard_info.is_null()) { fatal_error("cannot unpack a leaf entry from ShardHashes"); return; @@ -1498,14 +1495,14 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni LOG(INFO) << "performing a lookupBlock(" << blkid.to_str() << ", " << mode << ", " << lt << ", " << utime << ") query"; auto P = td::PromiseCreator::lambda( - [Self = actor_id(this), manager = manager_, mode = (mode >> 4)](td::Result res) { + [ Self = actor_id(this), manager = manager_, mode = (mode >> 4) ](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto handle = res.move_as_ok(); LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, - [Self, blkid = handle->id(), mode](td::Result> res) { + [ Self, blkid = handle->id(), mode ](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1653,7 +1650,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, if (mode & 0x1000) { BlockIdExt bblk = (from.seqno() > to.seqno()) ? from : to; td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, bblk, - [Self = actor_id(this), from, to, bblk, mode](td::Result> res) { + [ Self = actor_id(this), from, to, bblk, mode ](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1665,7 +1662,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, } else { td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [Self = actor_id(this), from, to, mode](td::Result, BlockIdExt>> res) { + [ Self = actor_id(this), from, to, mode ](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1678,7 +1675,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, } else if (mode & 2) { td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [Self = actor_id(this), from, mode](td::Result, BlockIdExt>> res) { + [ Self = actor_id(this), from, mode ](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1689,7 +1686,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, }); } else { td::actor::send_closure_later(manager_, &ton::validator::ValidatorManager::get_shard_client_state, false, - [Self = actor_id(this), from, mode](td::Result res) { + [ Self = actor_id(this), from, mode ](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 2adbc2eb..bcdc5050 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -160,7 +160,7 @@ class LiteQuery : public td::actor::Actor { const BlockIdExt& blkid); bool make_state_root_proof(Ref& proof, Ref state_root, Ref block_root, const BlockIdExt& blkid); - bool make_shard_info_proof(Ref& proof, vm::CellSlice& cs, ShardIdFull shard, ShardIdFull& true_shard, + bool make_shard_info_proof(Ref& proof, Ref& info, ShardIdFull shard, ShardIdFull& true_shard, Ref& leaf, bool& found, bool exact = true); bool make_shard_info_proof(Ref& proof, Ref& info, ShardIdFull shard, bool exact = true); bool make_shard_info_proof(Ref& proof, Ref& info, AccountIdPrefixFull prefix); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 80f3eb72..711c2e8c 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1210,7 +1210,7 @@ bool ValidateQuery::request_neighbor_queues() { } int i = 0; for (block::McShardDescr& descr : neighbors_) { - LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); + LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str(); ++pending; send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, [ self = get_self(), i ](td::Result> res) { @@ -3974,12 +3974,14 @@ bool ValidateQuery::check_delivered_dequeued() { // could look up neighbor with shard containing enq_msg_descr.next_prefix more efficiently // (instead of checking all neighbors) if (!nb.is_disabled() && nb.processed_upto->already_processed(enq)) { - // the message has been delivered! - return reject_query( - PSTRING() - << "outbound message with (lt,hash)=(" << enq.lt_ << "," << enq.hash_.to_hex() << ") enqueued_lt=" - << enq.enqueued_lt_ << " has been already delivered and processed by neighbor " << nb.blk_.to_str() - << " but it has not been dequeued in this block and it is still present in the new outbound queue"); + // the message has been delivered but not removed from queue! + LOG(WARNING) << "outbound queue not cleaned up completely (overfull block?): outbound message with (lt,hash)=(" + << enq.lt_ << "," << enq.hash_.to_hex() << ") enqueued_lt=" << enq.enqueued_lt_ + << " has been already delivered and processed by neighbor " << nb.blk_.to_str() + << " but it has not been dequeued in this block and it is still present in the new outbound queue"; + outq_cleanup_partial_ = true; + ok = true; + return false; // skip scanning the remainder of the queue } } if (created_lt >= start_lt_) { diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index c75b5877..f1c988d2 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -144,6 +144,7 @@ class ValidateQuery : public td::actor::Actor { bool is_fake_{false}; bool prev_key_block_exists_{false}; bool debug_checks_{false}; + bool outq_cleanup_partial_{false}; BlockSeqno prev_key_seqno_{~0u}; int stage_{0}; td::BitArray<64> shard_pfx_; diff --git a/validator/impl/validator-set.cpp b/validator/impl/validator-set.cpp index 733f28a2..629337cf 100644 --- a/validator/impl/validator-set.cpp +++ b/validator/impl/validator-set.cpp @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - Copyright 2017-2019 Telegram Systems LLP + Copyright 2017-2020 Telegram Systems LLP */ #include "validator-set.hpp" #include "auto/tl/ton_api.h" @@ -156,6 +156,8 @@ Ref ValidatorSetCompute::compute_validator_set(ShardIdFull shard, LOG(DEBUG) << "in compute_validator_set() for " << shard.to_str(); auto nodes = config_->compute_validator_set(shard, vset, time, ccseqno); if (nodes.empty()) { + LOG(ERROR) << "compute_validator_set() for " << shard.to_str() << "," << time << "," << ccseqno + << " returned empty list"; return {}; } return Ref{true, ccseqno, shard, std::move(nodes)}; diff --git a/validator/manager.cpp b/validator/manager.cpp index a9e59cef..dc75220a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1777,6 +1777,9 @@ void ValidatorManagerImpl::update_shards() { } for (auto &shard : future_shards) { auto val_set = last_masterchain_state_->get_next_validator_set(shard); + if (val_set.is_null()) { + continue; + } auto validator_id = get_validator(shard, val_set); if (!validator_id.is_zero()) { diff --git a/validator/token-manager.h b/validator/token-manager.h index 43e6d1ca..0d75710f 100644 --- a/validator/token-manager.h +++ b/validator/token-manager.h @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - Copyright 2017-2019 Telegram Systems LLP + Copyright 2017-2020 Telegram Systems LLP */ #pragma once @@ -55,9 +55,9 @@ class TokenManager : public td::actor::Actor { td::uint64 seqno_ = 0; std::map pending_; - td::uint32 free_tokens_ = 8; - td::uint32 free_priority_tokens_ = 8; - td::uint32 max_priority_tokens_ = 8; + td::uint32 free_tokens_ = 16; + td::uint32 free_priority_tokens_ = 16; + td::uint32 max_priority_tokens_ = 16; }; } // namespace validator