diff --git a/CMakeLists.txt b/CMakeLists.txt index a97a9c96..da1945ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -423,6 +423,9 @@ target_link_libraries(test-vm PRIVATE ton_crypto fift-lib) add_executable(test-smartcont test/test-td-main.cpp ${SMARTCONT_TEST_SOURCE}) target_link_libraries(test-smartcont PRIVATE smc-envelope fift-lib ton_db) +add_executable(test-bigint ${BIGINT_TEST_SOURCE}) +target_link_libraries(test-bigint PRIVATE ton_crypto) + add_executable(test-cells test/test-td-main.cpp ${CELLS_TEST_SOURCE}) target_link_libraries(test-cells PRIVATE ton_crypto) @@ -523,6 +526,7 @@ if (HAS_PARENT) ${FEC_TEST_SOURCE} ${ED25519_TEST_SOURCE} ${TONDB_TEST_SOURCE} + ${BIGNUM_TEST_SOURCE} ${CELLS_TEST_SOURCE} # ${TONVM_TEST_SOURCE} ${FIFT_TEST_SOURCE} ${TONLIB_ONLINE_TEST_SOURCE} PARENT_SCOPE) endif() @@ -536,6 +540,7 @@ set(TEST_OPTIONS "--regression ${CMAKE_CURRENT_SOURCE_DIR}/test/regression-tests separate_arguments(TEST_OPTIONS) add_test(test-ed25519-crypto crypto/test-ed25519-crypto) add_test(test-ed25519 test-ed25519) +add_test(test-bigint test-bigint) add_test(test-vm test-vm ${TEST_OPTIONS}) add_test(test-fift test-fift ${TEST_OPTIONS}) add_test(test-cells test-cells ${TEST_OPTIONS}) diff --git a/Changelog.md b/Changelog.md new file mode 100644 index 00000000..e9052c0b --- /dev/null +++ b/Changelog.md @@ -0,0 +1,14 @@ +## 05.2022 Update +* Initial synchronization improved: adjusted timeouts for state download and the way of choosing which state to download. Nodes with low network speed and/or bad connectivity will synchronize faster and consistently. +* Improved peer-to-peer network stability and DDoS resistance: now peers will only relay valid messages to the network. Large messages, which require splitting for relaying, will be retranslated as well, but only after the node gets all parts, and reassembles and checks them. Validators may sign certificates for network peers, which allow relaying large messages by parts without checks. It is used now by validators to faster relay new blocks. Sign and import certificate commands are exposed via `validator-engine-console`. +* Fixed some rare edge cases in TVM arithmetic operations related to big numbers (`2**63+`) +* Improved fixes used to combat wrong activate-destruct-activate contract behavior last November. +* Improved tonlib: support libraries (with client-side caching), getmethods completely fill c7 register, getmethods support slice arguments, improved messages listing for transactions, added extended block header params, added getConfig method. +* RocksDB updated to a newer version. +* Improved persistent state serialization: memory usage during serialization was optimized; the start of serialization on different nodes was sparsed. +* FunC update: support for string literals and constants (including precompiled constant expressions), semver, `include` expressions. +* Fixed rarely manifested bugs in `Asm.fif`. +* LiteClient supports key as cli parameter. +* Improved Liteserver DoS resistance for running getmethods. + +Besides the work of the core team, this update is based on the efforts of @tvorogme (added support for slice arguments and noted bugs in Asm.fif), @akifoq (fixed bug in Asm.fif), @cryshado (noted strange behavior of LS, which, upon inspection, turned out to be a vector of DoS attack). diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index b9b0373f..e7f5c019 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -486,7 +486,7 @@ void CatChainReceiverImpl::start_up() { } td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), - make_callback(), overlay::OverlayPrivacyRules{0, std::move(root_keys)}); + make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}); CHECK(root_block_); diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 79406ffc..76c8ae9d 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -259,6 +259,11 @@ set(FIFT_TEST_SOURCE PARENT_SCOPE ) +set(BIGINT_TEST_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/test/test-bigint.cpp + PARENT_SCOPE +) + add_library(ton_crypto STATIC ${TON_CRYPTO_SOURCE}) target_include_directories(ton_crypto PUBLIC $ @@ -293,9 +298,14 @@ add_library(src_parser ${PARSER_SOURCE}) target_include_directories(src_parser PUBLIC $) target_link_libraries(src_parser PUBLIC ton_crypto) +add_library(ton_block ${BLOCK_SOURCE}) +target_include_directories(ton_block PUBLIC $ + $ $) +target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api) + add_executable(func func/func.cpp ${FUNC_LIB_SOURCE}) target_include_directories(func PUBLIC $) -target_link_libraries(func PUBLIC ton_crypto src_parser git) +target_link_libraries(func PUBLIC ton_crypto src_parser git ton_block) if (WINGETOPT_FOUND) target_link_libraries_system(func wingetopt) endif() @@ -319,11 +329,6 @@ if (WINGETOPT_FOUND) target_link_libraries_system(pow-miner wingetopt) endif() -add_library(ton_block ${BLOCK_SOURCE}) -target_include_directories(ton_block PUBLIC $ - $ $) -target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api) - set(TURN_OFF_LSAN cd .) if (TON_USE_ASAN AND NOT WIN32) set(TURN_OFF_LSAN export LSAN_OPTIONS=detect_leaks=0) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 07574ef1..003cd04b 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2100,7 +2100,7 @@ Ref ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const { return {}; } auto csr = libraries_dict_->lookup(root_hash, 256); - if (csr.is_null() || csr->prefetch_ulong(8) != 0 || !csr->have_refs()) { // shared_lib_descr$00 lib:^Cell + if (csr.is_null() || csr->prefetch_ulong(2) != 0 || !csr->have_refs()) { // shared_lib_descr$00 lib:^Cell return {}; } auto lib = csr->prefetch_ref(); diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 3668a45f..e5cd701d 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -271,7 +271,7 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, } bool Account::init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite) { - if (split_depth_set_ || !created || !set_split_depth(split_depth)) { + if (split_depth_set_ || !set_split_depth(split_depth)) { return false; } addr_orig = addr; @@ -304,15 +304,8 @@ bool Account::unpack(Ref shard_account, Ref extra, total_state = orig_total_state = account; auto acc_cs = load_cell_slice(std::move(account)); if (block::gen::t_Account.get_tag(acc_cs) == block::gen::Account::account_none) { - status = acc_nonexist; - last_paid = 0; - last_trans_end_lt_ = 0; is_special = special; - if (workchain != ton::workchainInvalid) { - addr_orig = addr; - addr_rewrite = addr.cbits(); - } - return compute_my_addr() && acc_cs.size_ext() == 1; + return acc_cs.size_ext() == 1 && init_new(now); } block::gen::Account::Record_account acc; block::gen::AccountStorage::Record storage; @@ -328,6 +321,7 @@ bool Account::unpack(Ref shard_account, Ref extra, case block::gen::AccountState::account_uninit: status = orig_status = acc_uninit; state_hash = addr; + forget_split_depth(); break; case block::gen::AccountState::account_frozen: status = orig_status = acc_frozen; @@ -396,10 +390,42 @@ bool Account::init_new(ton::UnixTime now) { state_hash = addr_orig; status = orig_status = acc_nonexist; split_depth_set_ = false; - created = true; return true; } +bool Account::forget_split_depth() { + split_depth_set_ = false; + split_depth_ = 0; + addr_orig = addr; + my_addr = my_addr_exact; + addr_rewrite = addr.bits(); + return true; +} + +bool Account::deactivate() { + if (status == acc_active) { + return false; + } + // forget special (tick/tock) info + tick = tock = false; + if (status == acc_nonexist || status == acc_uninit) { + // forget split depth and address rewriting info + forget_split_depth(); + // forget specific state hash for deleted or uninitialized accounts (revert to addr) + state_hash = addr; + } + // forget code and data (only active accounts remember these) + code.clear(); + data.clear(); + library.clear(); + // if deleted, balance must be zero + if (status == acc_nonexist && !balance.is_zero()) { + return false; + } + return true; +} + + bool Account::belongs_to_shard(ton::ShardIdFull shard) const { return workchain == shard.workchain && ton::shard_is_ancestor(shard.shard, addr); } @@ -2214,7 +2240,7 @@ Ref Transaction::commit(Account& acc) { CHECK((const void*)&acc == (const void*)&account); // export all fields modified by the Transaction into original account // NB: this is the only method that modifies account - if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status == Account::acc_nonexist && + if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status != Account::acc_active && acc_status == Account::acc_active) { LOG(DEBUG) << "setting address rewriting info for newly-activated account " << acc.addr.to_hex() << " with split_depth=" << new_split_depth @@ -2243,9 +2269,7 @@ Ref Transaction::commit(Account& acc) { acc.tick = new_tick; acc.tock = new_tock; } else { - acc.tick = acc.tock = false; - acc.split_depth_set_ = false; - acc.created = true; + CHECK(acc.deactivate()); } end_lt = 0; acc.push_transaction(root, start_lt); diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index c1cfbc54..398e2731 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -213,17 +213,16 @@ struct Account { bool is_special{false}; bool tick{false}; bool tock{false}; - bool created{false}; bool split_depth_set_{false}; unsigned char split_depth_{0}; int verbosity{3 * 0}; ton::UnixTime now_{0}; ton::WorkchainId workchain{ton::workchainInvalid}; td::BitArray<32> addr_rewrite; // rewrite (anycast) data, split_depth bits - ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`) - ton::StdSmcAddress addr_orig; // address indicated in smart-contract data - Ref my_addr; // address as stored in the smart contract (MsgAddressInt) - Ref my_addr_exact; // exact address without anycast info + ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`); it is the key in ShardAccounts + ton::StdSmcAddress addr_orig; // address indicated in smart-contract data (must coincide with hash of StateInit) + Ref my_addr; // address as stored in the smart contract (MsgAddressInt); corresponds to `addr_orig` + anycast info + Ref my_addr_exact; // exact address without anycast info; corresponds to `addr` and has no anycast (rewrite) info ton::LogicalTime last_trans_end_lt_; ton::LogicalTime last_trans_lt_; ton::Bits256 last_trans_hash_; @@ -250,6 +249,7 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, Ref extra, ton::UnixTime now, bool special = false); bool init_new(ton::UnixTime now); + bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; bool is_masterchain() const { @@ -268,6 +268,7 @@ struct Account { friend struct Transaction; bool set_split_depth(int split_depth); bool check_split_depth(int split_depth) const; + bool forget_split_depth(); bool init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite); private: diff --git a/crypto/common/bigint.hpp b/crypto/common/bigint.hpp index e78f4892..94da3a13 100644 --- a/crypto/common/bigint.hpp +++ b/crypto/common/bigint.hpp @@ -264,7 +264,7 @@ class AnyIntView { return digits[size() - 1]; } double top_double() const { - return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * (1.0 / Tr::Base) + return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * Tr::InvBase : (double)digits[size() - 1]; } bool is_odd_any() const { @@ -314,8 +314,15 @@ class BigIntG { digits[0] = x; } BigIntG(Normalize, word_t x) : n(1) { - digits[0] = x; - normalize_bool(); + if (x >= -Tr::Half && x < Tr::Half) { + digits[0] = x; + } else if (len <= 1) { + digits[0] = x; + normalize_bool(); + } else { + digits[0] = ((x + Tr::Half) & (Tr::Base - 1)) - Tr::Half; + digits[n++] = (x >> Tr::word_shift) + (digits[0] < 0); + } } BigIntG(const BigIntG& x) : n(x.n) { std::memcpy(digits, x.digits, n * sizeof(word_t)); @@ -757,7 +764,7 @@ bool AnyIntView::add_pow2_any(int exponent, int factor) { while (size() <= k) { digits[inc_size()] = 0; } - digits[k] += (factor << dm.rem); + digits[k] += ((word_t)factor << dm.rem); return true; } @@ -1087,12 +1094,16 @@ int AnyIntView::cmp_any(const AnyIntView& yp) const { template int AnyIntView::cmp_any(word_t y) const { - if (size() > 1) { - return top_word() < 0 ? -1 : 1; - } else if (size() == 1) { + if (size() == 1) { return digits[0] < y ? -1 : (digits[0] > y ? 1 : 0); - } else { + } else if (!size()) { return 0x80000000; + } else if (size() == 2 && (y >= Tr::Half || y < -Tr::Half)) { + word_t x0 = digits[0] & (Tr::Base - 1), y0 = y & (Tr::Base - 1); + word_t x1 = digits[1] + (digits[0] >> Tr::word_shift), y1 = (y >> Tr::word_shift); + return x1 < y1 ? -1 : (x1 > y1 ? 1 : (x0 < y0 ? -1 : (x0 > y0 ? 1 : 0))); + } else { + return top_word() < 0 ? -1 : 1; } } @@ -1312,17 +1323,14 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, if (k > quot.max_size()) { return invalidate_bool(); } - quot.set_size(max(k,1)); - for(int qi=0; qi< max(k,1); qi++) { - quot.digits[qi]=0; - } + quot.set_size(std::max(k, 1)); + quot.digits[0] = 0; } else { if (k >= quot.max_size()) { return invalidate_bool(); } quot.set_size(k + 1); - double x_top = top_double(); - word_t q = std::llrint(x_top * y_inv * Tr::InvBase); + word_t q = std::llrint(top_double() * y_inv * Tr::InvBase); quot.digits[k] = q; int i = yp.size() - 1; word_t hi = 0; @@ -1337,8 +1345,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, quot.digits[0] = 0; } while (--k >= 0) { - double x_top = top_double(); - word_t q = std::llrint(x_top * y_inv); + word_t q = std::llrint(top_double() * y_inv); quot.digits[k] = q; for (int i = yp.size() - 1; i >= 0; --i) { Tr::sub_mul(&digits[k + i + 1], &digits[k + i], q, yp.digits[i]); @@ -1346,15 +1353,18 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, dec_size(); digits[size() - 1] += (digits[size()] << word_shift); } - if (size() >= yp.size()) { - assert(size() == yp.size()); - double x_top = top_double(); - double t = x_top * y_inv * Tr::InvBase; + if (size() >= yp.size() - 1) { + assert(size() <= yp.size()); + bool grow = (size() < yp.size()); + double t = top_double() * y_inv * (grow ? Tr::InvBase * Tr::InvBase : Tr::InvBase); if (round_mode >= 0) { t += (round_mode ? 1 : 0.5); } word_t q = std::llrint(std::floor(t)); if (q) { + if (grow) { + digits[inc_size()] = 0; + } for (int i = 0; i < size(); i++) { digits[i] -= q * yp.digits[i]; } @@ -1411,6 +1421,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, return normalize_bool_any(); } +// works for almost-normalized numbers (digits -Base+1 .. Base-1, top non-zero), result also almost-normalized template bool AnyIntView::mod_pow2_any(int exponent) { if (!is_valid()) { @@ -1462,25 +1473,21 @@ bool AnyIntView::mod_pow2_any(int exponent) { if (exponent >= max_size() * word_shift) { return invalidate_bool(); } - if (q - word_shift >= 0) { + if (q - word_shift >= 0) { // original top digit was a non-zero multiple of Base, impossible(?) digits[size() - 1] = 0; digits[inc_size()] = ((word_t)1 << (q - word_shift)); - } - if (q - word_shift == -1 && size() < max_size() - 1) { + } else if (q - word_shift == -1 && size() < max_size()) { digits[size() - 1] = -Tr::Half; digits[inc_size()] = 1; } else { digits[size() - 1] = pow; } return true; - } else if (v >= Tr::Half) { - if (size() == max_size() - 1) { - return invalidate_bool(); - } else { - digits[size() - 1] = v | -Tr::Half; - digits[inc_size()] = ((word_t)1 << (q - word_shift)); - return true; - } + } else if (v >= Tr::Half && size() < max_size()) { + word_t w = (((v >> (word_shift - 1)) + 1) >> 1); + digits[size() - 1] = v - (w << word_shift); + digits[inc_size()] = w; + return true; } else { digits[size() - 1] = v; return true; diff --git a/crypto/common/refint.cpp b/crypto/common/refint.cpp index b79750ce..8e06da5f 100644 --- a/crypto/common/refint.cpp +++ b/crypto/common/refint.cpp @@ -128,12 +128,10 @@ RefInt256 muldiv(RefInt256 x, RefInt256 y, RefInt256 z, int round_mode) { } std::pair muldivmod(RefInt256 x, RefInt256 y, RefInt256 z, int round_mode) { - typename td::BigInt256::DoubleInt tmp{0}; + typename td::BigInt256::DoubleInt tmp{0}, quot; tmp.add_mul(*x, *y); - RefInt256 quot{true}; - tmp.mod_div(*z, quot.unique_write(), round_mode); - quot.write().normalize(); - return std::make_pair(std::move(quot), td::make_refint(tmp)); + tmp.mod_div(*z, quot, round_mode); + return std::make_pair(td::make_refint(quot.normalize()), td::make_refint(tmp)); } RefInt256 operator&(RefInt256 x, RefInt256 y) { diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index d61923c1..71c6a956 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -334,9 +334,14 @@ x{A926} @Defop RSHIFTC x{A935} @Defop(8u+1) RSHIFTR# x{A936} @Defop(8u+1) RSHIFTC# x{A938} @Defop(8u+1) MODPOW2# +x{A939} @Defop(8u+1) MODPOW2R# +x{A93A} @Defop(8u+1) MODPOW2C# x{A984} @Defop MULDIV x{A985} @Defop MULDIVR +x{A988} @Defop MULMOD x{A98C} @Defop MULDIVMOD +x{A98D} @Defop MULDIVMODR +x{A98E} @Defop MULDIVMODC x{A9A4} @Defop MULRSHIFT x{A9A5} @Defop MULRSHIFTR x{A9A6} @Defop MULRSHIFTC @@ -649,7 +654,7 @@ x{DB32} dup @Defop BRANCH @Defop RETBOOL x{DB34} @Defop CALLCC x{DB35} @Defop JMPXDATA { dup 1+ 0= { 16 + } if - & vars, std::string pfx show_var_list(os, left, vars); os << " := " << int_const << std::endl; break; + case _SliceConst: + os << pfx << dis << "SCONST "; + show_var_list(os, left, vars); + os << " := " << str_const << std::endl; + break; case _Import: os << pfx << dis << "IMPORT "; show_var_list(os, left, vars); diff --git a/crypto/func/analyzer.cpp b/crypto/func/analyzer.cpp index 5af435bc..26162521 100644 --- a/crypto/func/analyzer.cpp +++ b/crypto/func/analyzer.cpp @@ -353,6 +353,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { } switch (cl) { case _IntConst: + case _SliceConst: case _GlobVar: case _Call: case _CallInd: @@ -540,6 +541,7 @@ bool prune_unreachable(std::unique_ptr& ops) { bool reach; switch (op.cl) { case Op::_IntConst: + case Op::_SliceConst: case Op::_GlobVar: case Op::_SetGlob: case Op::_Call: @@ -707,6 +709,10 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { values.add_newval(left[0]).set_const(int_const); break; } + case _SliceConst: { + values.add_newval(left[0]).set_const(str_const); + break; + } case _Call: { prepare_args(values); auto func = dynamic_cast(fun_ref->value); @@ -848,6 +854,7 @@ bool Op::mark_noreturn() { // fallthrough case _Import: case _IntConst: + case _SliceConst: case _Let: case _Tuple: case _UnTuple: diff --git a/crypto/func/asmops.cpp b/crypto/func/asmops.cpp index f0df9f3d..ccc409d3 100644 --- a/crypto/func/asmops.cpp +++ b/crypto/func/asmops.cpp @@ -55,10 +55,10 @@ std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg) { } } -AsmOp AsmOp::Const(int arg, std::string push_op) { +AsmOp AsmOp::Const(int arg, std::string push_op, td::RefInt256 origin) { std::ostringstream os; os << arg << ' ' << push_op; - return AsmOp::Const(os.str()); + return AsmOp::Const(os.str(), origin); } AsmOp AsmOp::make_stk2(int a, int b, const char* str, int delta) { @@ -166,27 +166,27 @@ AsmOp AsmOp::UnTuple(int a) { AsmOp AsmOp::IntConst(td::RefInt256 x) { if (x->signed_fits_bits(8)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT"); + return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); } if (!x->is_valid()) { - return AsmOp::Const("PUSHNAN"); + return AsmOp::Const("PUSHNAN", x); } int k = is_pos_pow2(x); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2"); + return AsmOp::Const(k, "PUSHPOW2", x); } k = is_pos_pow2(x + 1); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2DEC"); + return AsmOp::Const(k, "PUSHPOW2DEC", x); } k = is_pos_pow2(-x); if (k >= 0) { - return AsmOp::Const(k, "PUSHNEGPOW2"); + return AsmOp::Const(k, "PUSHNEGPOW2", x); } if (!x->mod_pow2_short(23)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX"); + return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX", x); } - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT"); + return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); } AsmOp AsmOp::BoolConst(bool f) { diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index 91020be0..075d8f62 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -27,7 +27,7 @@ using namespace std::literals::string_literals; * */ -int glob_func_cnt, undef_func_cnt, glob_var_cnt; +int glob_func_cnt, undef_func_cnt, glob_var_cnt, const_cnt; std::vector glob_func, glob_vars; SymDef* predefine_builtin_func(std::string name, TypeExpr* func_type) { diff --git a/crypto/func/codegen.cpp b/crypto/func/codegen.cpp index 26d8215e..3e58b995 100644 --- a/crypto/func/codegen.cpp +++ b/crypto/func/codegen.cpp @@ -302,6 +302,15 @@ bool Op::generate_code_step(Stack& stack) { } return true; } + case _SliceConst: { + auto p = next_var_info[left[0]]; + if (!p || p->is_unused()) { + return true; + } + stack.o << AsmOp::Const("x{" + str_const + "} PUSHSLICE"); + stack.push_new_var(left[0]); + return true; + } case _GlobVar: if (dynamic_cast(fun_ref->value)) { bool used = false; diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index e8e09961..b260c885 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -173,16 +173,29 @@ void usage(const char* progname) { "-R\tInclude operation rewrite comments in the output code\n" "-W\tInclude Fift code to serialize and save generated code into specified BoC file. Enables " "-A and -P.\n" + "\t-s\tOutput semantic version of FunC and exit\n" "\t-V\tShow func build information\n"; std::exit(2); } +void output_inclusion_stack() { + while (!funC::inclusion_locations.empty()) { + src::SrcLocation loc = funC::inclusion_locations.top(); + funC::inclusion_locations.pop(); + if (loc.fdescr) { + std::cerr << "note: included from "; + loc.show(std::cerr); + std::cerr << std::endl; + } + } +} + std::string output_filename; int main(int argc, char* const argv[]) { int i; bool interactive = false; - while ((i = getopt(argc, argv, "Ahi:Io:O:PRSvW:V")) != -1) { + while ((i = getopt(argc, argv, "Ahi:Io:O:PRsSvW:V")) != -1) { switch (i) { case 'A': funC::asm_preamble = true; @@ -215,8 +228,13 @@ int main(int argc, char* const argv[]) { funC::boc_output_filename = optarg; funC::asm_preamble = funC::program_envelope = true; break; + case 's': + std::cout << funC::func_version << "\n"; + std::exit(0); + break; case 'V': - std::cout << "Func build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "FunC semantic version: v" << funC::func_version << "\n"; + std::cout << "Build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); break; case 'h': @@ -235,7 +253,7 @@ int main(int argc, char* const argv[]) { int ok = 0, proc = 0; try { while (optind < argc) { - funC::generated_from += std::string{"`"} + argv[optind] + "` "; + // funC::generated_from += std::string{"`"} + argv[optind] + "` "; ok += funC::parse_source_file(argv[optind++]); proc++; } @@ -262,14 +280,17 @@ int main(int argc, char* const argv[]) { funC::generate_output(); } catch (src::Fatal& fatal) { std::cerr << "fatal: " << fatal << std::endl; + output_inclusion_stack(); std::exit(1); } catch (src::Error& error) { std::cerr << error << std::endl; + output_inclusion_stack(); std::exit(1); } catch (funC::UnifyError& unif_err) { std::cerr << "fatal: "; unif_err.print_message(std::cerr); std::cerr << std::endl; + output_inclusion_stack(); std::exit(1); } } diff --git a/crypto/func/func.h b/crypto/func/func.h index 1506b974..2f9877a0 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -35,9 +35,12 @@ namespace funC { extern int verbosity; extern bool op_rewrite_comments; +extern std::string generated_from; constexpr int optimize_depth = 20; +const std::string func_version{"0.2.0"}; + enum Keyword { _Eof = -1, _Ident = 0, @@ -106,7 +109,10 @@ enum Keyword { _Operator, _Infix, _Infixl, - _Infixr + _Infixr, + _Const, + _PragmaHashtag, + _IncludeHashtag }; void define_keywords(); @@ -333,6 +339,8 @@ struct VarDescr { static constexpr int FiniteUInt = FiniteInt | _Pos; int val; td::RefInt256 int_const; + std::string str_const; + VarDescr(var_idx_t _idx = -1, int _flags = 0, int _val = 0) : idx(_idx), flags(_flags), val(_val) { } bool operator<(var_idx_t other_idx) const { @@ -403,6 +411,7 @@ struct VarDescr { } void set_const(long long value); void set_const(td::RefInt256 value); + void set_const(std::string value); void set_const_nan(); void operator+=(const VarDescr& y) { flags &= y.flags; @@ -527,7 +536,8 @@ struct Op { _While, _Until, _Repeat, - _Again + _Again, + _SliceConst }; int cl; enum { _Disabled = 1, _Reachable = 2, _NoReturn = 4, _ImpureR = 8, _ImpureW = 16, _Impure = 24 }; @@ -540,6 +550,7 @@ struct Op { std::vector left, right; std::unique_ptr block0, block1; td::RefInt256 int_const; + std::string str_const; Op(const SrcLocation& _where = {}, int _cl = _Undef) : cl(_cl), flags(0), fun_ref(nullptr), where(_where) { } Op(const SrcLocation& _where, int _cl, const std::vector& _left) @@ -551,6 +562,9 @@ struct Op { Op(const SrcLocation& _where, int _cl, const std::vector& _left, td::RefInt256 _const) : cl(_cl), flags(0), fun_ref(nullptr), where(_where), left(_left), int_const(_const) { } + Op(const SrcLocation& _where, int _cl, const std::vector& _left, std::string _const) + : cl(_cl), flags(0), fun_ref(nullptr), where(_where), left(_left), str_const(_const) { + } Op(const SrcLocation& _where, int _cl, const std::vector& _left, const std::vector& _right, SymDef* _fun = nullptr) : cl(_cl), flags(0), fun_ref(_fun), where(_where), left(_left), right(_right) { @@ -781,6 +795,30 @@ struct SymValGlobVar : sym::SymValBase { } }; +struct SymValConst : sym::SymValBase { + td::RefInt256 intval; + std::string strval; + Keyword type; + SymValConst(int idx, td::RefInt256 value) + : sym::SymValBase(_Const, idx), intval(value) { + type = _Int; + } + SymValConst(int idx, std::string value) + : sym::SymValBase(_Const, idx), strval(value) { + type = _Slice; + } + ~SymValConst() override = default; + td::RefInt256 get_int_value() const { + return intval; + } + std::string get_str_value() const { + return strval; + } + Keyword get_type() const { + return type; + } +}; + extern int glob_func_cnt, undef_func_cnt, glob_var_cnt; extern std::vector glob_func, glob_vars; @@ -792,9 +830,11 @@ extern std::vector glob_func, glob_vars; // defined in parse-func.cpp bool parse_source(std::istream* is, const src::FileDescr* fdescr); -bool parse_source_file(const char* filename); +bool parse_source_file(const char* filename, src::Lexem lex = {}); bool parse_source_stdin(); +extern std::stack inclusion_locations; + /* * * EXPRESSIONS @@ -817,7 +857,8 @@ struct Expr { _LetFirst, _Hole, _Type, - _CondExpr + _CondExpr, + _SliceConst }; int cls; int val{0}; @@ -825,6 +866,7 @@ struct Expr { int flags{0}; SrcLocation here; td::RefInt256 intval; + std::string strval; SymDef* sym{nullptr}; TypeExpr* e_type{nullptr}; std::vector args; @@ -907,6 +949,7 @@ struct AsmOp { int a, b, c; bool gconst{false}; std::string op; + td::RefInt256 origin; struct SReg { int idx; SReg(int _idx) : idx(_idx) { @@ -926,6 +969,9 @@ struct AsmOp { AsmOp(int _t, int _a, int _b, std::string _op) : t(_t), a(_a), b(_b), op(std::move(_op)) { compute_gconst(); } + AsmOp(int _t, int _a, int _b, std::string _op, td::RefInt256 x) : t(_t), a(_a), b(_b), op(std::move(_op)), origin(x) { + compute_gconst(); + } AsmOp(int _t, int _a, int _b, int _c) : t(_t), a(_a), b(_b), c(_c) { } AsmOp(int _t, int _a, int _b, int _c, std::string _op) : t(_t), a(_a), b(_b), c(_c), op(std::move(_op)) { @@ -1044,10 +1090,10 @@ struct AsmOp { static AsmOp make_stk3(int a, int b, int c, const char* str, int delta); static AsmOp IntConst(td::RefInt256 value); static AsmOp BoolConst(bool f); - static AsmOp Const(std::string push_op) { - return AsmOp(a_const, 0, 1, std::move(push_op)); + static AsmOp Const(std::string push_op, td::RefInt256 origin = {}) { + return AsmOp(a_const, 0, 1, std::move(push_op), origin); } - static AsmOp Const(int arg, std::string push_op); + static AsmOp Const(int arg, std::string push_op, td::RefInt256 origin = {}); static AsmOp Comment(std::string comment) { return AsmOp(a_none, std::string{"// "} + comment); } diff --git a/crypto/func/gen-abscode.cpp b/crypto/func/gen-abscode.cpp index c0a8985a..21dce714 100644 --- a/crypto/func/gen-abscode.cpp +++ b/crypto/func/gen-abscode.cpp @@ -362,6 +362,11 @@ std::vector Expr::pre_compile(CodeBlob& code, bool lval) const { code.close_pop_cur(args[2]->here); return rvect; } + case _SliceConst: { + auto rvect = new_tmp_vect(code); + code.emplace_back(here, Op::_SliceConst, rvect, strval); + return rvect; + } default: std::cerr << "expression constructor is " << cls << std::endl; throw src::Fatal{"cannot compile expression with unknown constructor"}; diff --git a/crypto/func/keywords.cpp b/crypto/func/keywords.cpp index 47975e7f..3f0fa230 100644 --- a/crypto/func/keywords.cpp +++ b/crypto/func/keywords.cpp @@ -125,7 +125,11 @@ void define_keywords() { .add_keyword("operator", Kw::_Operator) .add_keyword("infix", Kw::_Infix) .add_keyword("infixl", Kw::_Infixl) - .add_keyword("infixr", Kw::_Infixr); + .add_keyword("infixr", Kw::_Infixr) + .add_keyword("const", Kw::_Const); + + sym::symbols.add_keyword("#pragma", Kw::_PragmaHashtag) + .add_keyword("#include", Kw::_IncludeHashtag); } } // namespace funC diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 3aad90ef..0d2aa985 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -19,7 +19,11 @@ #include "func.h" #include "td/utils/crypto.h" #include "common/refint.h" +#include "openssl/digest.hpp" +#include "block/block.h" +#include "block-parse.h" #include +#include "td/utils/port/path.h" namespace sym { @@ -229,6 +233,83 @@ void parse_global_var_decl(Lexer& lex) { lex.next(); } +extern int const_cnt; +Expr* parse_expr(Lexer& lex, CodeBlob& code, bool nv = false); + +void parse_const_decl(Lexer& lex) { + SrcLocation loc = lex.cur().loc; + int wanted_type = Expr::_None; + if (lex.tp() == _Int) { + wanted_type = Expr::_Const; + lex.next(); + } else if (lex.tp() == _Slice) { + wanted_type = Expr::_SliceConst; + lex.next(); + } + if (lex.tp() != _Ident) { + lex.expect(_Ident, "constant name"); + } + loc = lex.cur().loc; + SymDef* sym_def = sym::define_global_symbol(lex.cur().val, false, loc); + if (!sym_def) { + lex.cur().error_at("cannot define global symbol `", "`"); + } + if (sym_def->value) { + lex.cur().error_at("global symbol `", "` already exists"); + } + lex.next(); + if (lex.tp() != '=') { + lex.cur().error_at("expected = instead of ", ""); + } + lex.next(); + CodeBlob code; + // Handles processing and resolution of literals and consts + auto x = parse_expr(lex, code, false); // also does lex.next() ! + if (x->flags != Expr::_IsRvalue) { + lex.cur().error("expression is not strictly Rvalue"); + } + if ((wanted_type == Expr::_Const) && (x->cls == Expr::_Apply)) + wanted_type = Expr::_None; // Apply is additionally checked to result in an integer + if ((wanted_type != Expr::_None) && (x->cls != wanted_type)) { + lex.cur().error("expression type does not match wanted type"); + } + if (x->cls == Expr::_Const) { // Integer constant + sym_def->value = new SymValConst{const_cnt++, x->intval}; + } else if (x->cls == Expr::_SliceConst) { // Slice constant (string) + sym_def->value = new SymValConst{const_cnt++, x->strval}; + } else if (x->cls == Expr::_Apply) { + code.emplace_back(loc, Op::_Import, std::vector()); + auto tmp_vars = x->pre_compile(code); + code.emplace_back(loc, Op::_Return, std::move(tmp_vars)); + code.emplace_back(loc, Op::_Nop); // This is neccessary to prevent SIGSEGV! + // It is REQUIRED to execute "optimizations" as in func.cpp + code.simplify_var_types(); + code.prune_unreachable_code(); + code.split_vars(true); + for (int i = 0; i < 16; i++) { + code.compute_used_code_vars(); + code.fwd_analyze(); + code.prune_unreachable_code(); + } + code.mark_noreturn(); + AsmOpList out_list(0, &code.vars); + code.generate_code(out_list); + if (out_list.list_.size() != 1) { + lex.cur().error("precompiled expression must result in single operation"); + } + auto op = out_list.list_[0]; + if (!op.is_const()) { + lex.cur().error("precompiled expression must result in compilation time constant"); + } + if (op.origin.is_null() || !op.origin->is_valid()) { + lex.cur().error("precompiled expression did not result in a valid integer constant"); + } + sym_def->value = new SymValConst{const_cnt++, op.origin}; + } else { + lex.cur().error("integer or slice literal or constant expected"); + } +} + FormalArgList parse_formal_args(Lexer& lex) { FormalArgList args; lex.expect('(', "formal argument list"); @@ -246,6 +327,18 @@ FormalArgList parse_formal_args(Lexer& lex) { return args; } +void parse_const_decls(Lexer& lex) { + lex.expect(_Const); + while (true) { + parse_const_decl(lex); + if (lex.tp() != ',') { + break; + } + lex.expect(','); + } + lex.expect(';'); +} + TypeExpr* extract_total_arg_type(const FormalArgList& arg_list) { if (arg_list.empty()) { return TypeExpr::new_unit(); @@ -322,8 +415,6 @@ Expr* make_func_apply(Expr* fun, Expr* x) { return res; } -Expr* parse_expr(Lexer& lex, CodeBlob& code, bool nv = false); - // parse ( E { , E } ) | () | [ E { , E } ] | [] | id | num | _ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { if (lex.tp() == '(' || lex.tp() == '[') { @@ -388,6 +479,87 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { lex.next(); return res; } + if (t == Lexem::String) { + std::string str = lex.cur().str; + int str_type = lex.cur().val; + Expr* res; + switch (str_type) { + case 0: + case 's': + case 'a': + { + res = new Expr{Expr::_SliceConst, lex.cur().loc}; + res->e_type = TypeExpr::new_atomic(_Slice); + break; + } + case 'u': + case 'h': + case 'H': + case 'c': + { + res = new Expr{Expr::_Const, lex.cur().loc}; + res->e_type = TypeExpr::new_atomic(_Int); + break; + } + default: + { + res = new Expr{Expr::_Const, lex.cur().loc}; + res->e_type = TypeExpr::new_atomic(_Int); + lex.cur().error("invalid string type `" + std::string(1, static_cast(str_type)) + "`"); + return res; + } + } + res->flags = Expr::_IsRvalue; + switch (str_type) { + case 0: { + res->strval = td::hex_encode(str); + break; + } + case 's': { + res->strval = str; + unsigned char buff[128]; + int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); + if (bits < 0) { + lex.cur().error_at("Invalid hex bitstring constant `", "`"); + } + break; + } + case 'a': { // MsgAddressInt + block::StdAddress a; + if (a.parse_addr(str)) { + res->strval = block::tlb::MsgAddressInt().pack_std_address(a)->as_bitslice().to_hex(); + } else { + lex.cur().error_at("invalid standard address `", "`"); + } + break; + } + case 'u': { + res->intval = td::hex_string_to_int256(td::hex_encode(str)); + if (!str.size()) { + lex.cur().error("empty integer ascii-constant"); + } + if (res->intval.is_null()) { + lex.cur().error_at("too long integer ascii-constant `", "`"); + } + break; + } + case 'h': + case 'H': + { + unsigned char hash[32]; + digest::hash_str(hash, str.data(), str.size()); + res->intval = td::bits_to_refint(hash, (str_type == 'h') ? 32 : 256, false); + break; + } + case 'c': + { + res->intval = td::make_refint(td::crc32(td::Slice{str})); + break; + } + } + lex.next(); + return res; + } if (t == '_') { Expr* res = new Expr{Expr::_Hole, lex.cur().loc}; res->val = -1; @@ -429,6 +601,25 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { lex.next(); return res; } + if (sym && dynamic_cast(sym->value)) { + auto val = dynamic_cast(sym->value); + Expr* res = new Expr{Expr::_None, lex.cur().loc}; + res->flags = Expr::_IsRvalue; + if (val->type == _Int) { + res->cls = Expr::_Const; + res->intval = val->get_int_value(); + } + else if (val->type == _Slice) { + res->cls = Expr::_SliceConst; + res->strval = val->get_str_value(); + } + else { + lex.cur().error("Invalid symbolic constant type"); + } + res->e_type = TypeExpr::new_atomic(val->type); + lex.next(); + return res; + } bool auto_apply = false; Expr* res = new Expr{Expr::_Var, lex.cur().loc}; if (nv) { @@ -1288,14 +1479,168 @@ void parse_func_def(Lexer& lex) { sym::close_scope(lex); } +std::string func_ver_test = func_version; + +void parse_pragma(Lexer& lex) { + auto pragma = lex.cur(); + lex.next(); + if (lex.tp() != _Ident) { + lex.expect(_Ident, "pragma name expected"); + } + auto pragma_name = lex.cur().str; + lex.next(); + if (!pragma_name.compare("version") || !pragma_name.compare("not-version")) { + bool negate = !pragma_name.compare("not-version"); + char op = '='; bool eq = false; + int sem_ver[3] = {0, 0, 0}; + char segs = 1; + if (lex.tp() == _Number) { + sem_ver[0] = std::stoi(lex.cur().str); + } else if (lex.tp() == _Ident) { + auto id1 = lex.cur().str; + char ch1 = id1[0]; + if ((ch1 == '>') || (ch1 == '<') || (ch1 == '=') || (ch1 == '^')) { + op = ch1; + } else { + lex.cur().error("unexpected comparator operation"); + } + if (id1.length() < 2) { + lex.cur().error("expected number after comparator"); + } + if (id1[1] == '=') { + eq = true; + if (id1.length() < 3) { + lex.cur().error("expected number after comparator"); + } + sem_ver[0] = std::stoi(id1.substr(2)); + } else { + sem_ver[0] = std::stoi(id1.substr(1)); + } + } else { + lex.cur().error("expected semver with optional comparator"); + } + lex.next(); + if (lex.tp() != ';') { + if (lex.tp() != _Ident || lex.cur().str[0] != '.') { + lex.cur().error("invalid semver format"); + } + sem_ver[1] = std::stoi(lex.cur().str.substr(1)); + segs = 2; + lex.next(); + } + if (lex.tp() != ';') { + if (lex.tp() != _Ident || lex.cur().str[0] != '.') { + lex.cur().error("invalid semver format"); + } + sem_ver[2] = std::stoi(lex.cur().str.substr(1)); + segs = 3; + lex.next(); + } + // End reading semver from source code + int func_ver[3] = {0, 0, 0}; + std::istringstream iss(func_ver_test); + std::string s; + for (int idx = 0; idx < 3; idx++) { + std::getline(iss, s, '.'); + func_ver[idx] = std::stoi(s); + } + // End parsing embedded semver + std::string semver_expr; + if (negate) { + semver_expr += '!'; + } + semver_expr += op; + if (eq) { + semver_expr += '='; + } + for (int idx = 0; idx < 3; idx++) { + semver_expr += std::to_string(sem_ver[idx]); + if (idx < 2) + semver_expr += '.'; + } + bool match = true; + switch (op) { + case '=': + if ((func_ver[0] != sem_ver[0]) || + (func_ver[1] != sem_ver[1]) || + (func_ver[2] != sem_ver[2])) { + match = false; + } + break; + case '>': + if ( ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] < sem_ver[2])) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] < sem_ver[1])) || + ((func_ver[0] < sem_ver[0])) ) { + match = false; + } + break; + case '<': + if ( ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] > sem_ver[2])) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] > sem_ver[1])) || + ((func_ver[0] > sem_ver[0])) ) { + match = false; + } + break; + case '^': + if ( ((segs == 3) && ((func_ver[0] != sem_ver[0]) || (func_ver[1] != sem_ver[1]) || (func_ver[2] < sem_ver[2]))) + || ((segs == 2) && ((func_ver[0] != sem_ver[0]) || (func_ver[1] < sem_ver[1]))) + || ((segs == 1) && ((func_ver[0] < sem_ver[0]))) ) { + match = false; + } + break; + } + if ((match && negate) || (!match && !negate)) { + pragma.error(std::string("FunC version ") + func_ver_test + " does not satisfy condition " + semver_expr); + } + } else if (!pragma_name.compare("test-version-set")) { + if (lex.tp() != _String) { + lex.cur().error("version string expected"); + } + func_ver_test = lex.cur().str; + lex.next(); + } else { + lex.cur().error(std::string{"unknown pragma `"} + pragma_name + "`"); + } + lex.expect(';'); +} + std::vector source_fdescr; +std::vector source_files; +std::stack inclusion_locations; + +void parse_include(Lexer& lex, const src::FileDescr* fdescr) { + auto include = lex.cur(); + lex.expect(_IncludeHashtag); + if (lex.tp() != _String) { + lex.expect(_String, "source file name"); + } + std::string val = lex.cur().str; + std::string parent_dir = fdescr->filename; + if (parent_dir.rfind('/') != std::string::npos) { + val = parent_dir.substr(0, parent_dir.rfind('/') + 1) + val; + } + lex.next(); + lex.expect(';'); + if (!parse_source_file(val.c_str(), include)) { + include.error(std::string{"failed parsing included file `"} + val + "`"); + } +} + bool parse_source(std::istream* is, src::FileDescr* fdescr) { src::SourceReader reader{is, fdescr}; Lexer lex{reader, true, ";,()[] ~."}; while (lex.tp() != _Eof) { - if (lex.tp() == _Global) { + if (lex.tp() == _PragmaHashtag) { + parse_pragma(lex); + } else if (lex.tp() == _IncludeHashtag) { + parse_include(lex, fdescr); + } else if (lex.tp() == _Global) { parse_global_var_decls(lex); + } else if (lex.tp() == _Const) { + parse_const_decls(lex); } else { parse_func_def(lex); } @@ -1303,17 +1648,46 @@ bool parse_source(std::istream* is, src::FileDescr* fdescr) { return true; } -bool parse_source_file(const char* filename) { +bool parse_source_file(const char* filename, src::Lexem lex) { if (!filename || !*filename) { - throw src::Fatal{"source file name is an empty string"}; + auto msg = "source file name is an empty string"; + if (lex.tp) { + lex.error(msg); + } else { + throw src::Fatal{msg}; + } } + std::string real_filename = td::realpath(td::CSlice(filename)).move_as_ok(); + if (std::count(source_files.begin(), source_files.end(), real_filename)) { + if (verbosity >= 2) { + if (lex.tp) { + lex.loc.show_warning(std::string{"skipping file "} + real_filename + " because it was already included"); + } else { + std::cerr << "warning: skipping file " << real_filename << " because it was already included" << std::endl; + } + } + return true; + } + if (lex.tp) { // included + funC::generated_from += std::string{"incl:"}; + } + funC::generated_from += std::string{"`"} + filename + "` "; + source_files.push_back(real_filename); src::FileDescr* cur_source = new src::FileDescr{filename}; source_fdescr.push_back(cur_source); std::ifstream ifs{filename}; if (ifs.fail()) { - throw src::Fatal{std::string{"cannot open source file `"} + filename + "`"}; + auto msg = std::string{"cannot open source file `"} + filename + "`"; + if (lex.tp) { + lex.error(msg); + } else { + throw src::Fatal{msg}; + } } - return parse_source(&ifs, cur_source); + inclusion_locations.push(lex.loc); + bool res = parse_source(&ifs, cur_source); + inclusion_locations.pop(); + return res; } bool parse_source_stdin() { diff --git a/crypto/func/test/co1.fc b/crypto/func/test/co1.fc new file mode 100644 index 00000000..82071e45 --- /dev/null +++ b/crypto/func/test/co1.fc @@ -0,0 +1,55 @@ +const int1 = 1, int2 = 2; + +const int int101 = 101; +const int int111 = 111; + +const int1r = int1; + +const str1 = "const1", str2 = "aabbcc"s; + +const slice str2r = str2; + +const str1int = 0x636f6e737431; +const str2int = 0xAABBCC; + +const int nibbles = 4; + +int iget1() { return int1; } +int iget2() { return int2; } +int iget3() { return int1 + int2; } + +int iget1r() { return int1r; } + +slice sget1() { return str1; } +slice sget2() { return str2; } +slice sget2r() { return str2r; } + +const int int240 = ((int1 + int2) * 10) << 3; + +int iget240() { return int240; } + +builder newc() asm "NEWC"; +slice endcs(builder b) asm "ENDC" "CTOS"; +int sdeq (slice s1, slice s2) asm "SDEQ"; +builder stslicer(builder b, slice s) asm "STSLICER"; + +_ main() { + int i1 = iget1(); + int i2 = iget2(); + int i3 = iget3(); + + throw_unless(int101, i1 == 1); + throw_unless(102, i2 == 2); + throw_unless(103, i3 == 3); + + slice s1 = sget1(); + slice s2 = sget2(); + slice s3 = newc().stslicer(str1).stslicer(str2r).endcs(); + + throw_unless(int111, sdeq(s1, newc().store_uint(str1int, 12 * nibbles).endcs())); + throw_unless(112, sdeq(s2, newc().store_uint(str2int, 6 * nibbles).endcs())); + throw_unless(113, sdeq(s3, newc().store_uint(0x636f6e737431ABCDEF, 18 * nibbles).endcs())); + + int i4 = iget240(); + throw_unless(104, i4 == 240); +} diff --git a/crypto/func/test/i1.fc b/crypto/func/test/i1.fc new file mode 100644 index 00000000..04f7889e --- /dev/null +++ b/crypto/func/test/i1.fc @@ -0,0 +1,16 @@ +global int i; + +#include "i1sub1.fc"; + +() sub0() impure { i = 0; } + +#include "i1sub2.fc"; + +() main() impure { + sub0(); + sub1(); + sub2(); + i = 9; +} + +#include "../test/i1sub2.fc"; \ No newline at end of file diff --git a/crypto/func/test/i1sub1.fc b/crypto/func/test/i1sub1.fc new file mode 100644 index 00000000..62bdd177 --- /dev/null +++ b/crypto/func/test/i1sub1.fc @@ -0,0 +1,8 @@ +;; DO NOT COMPILE DIRECTLY! +;; Compile i1.fc + +#include "i1sub1.fc"; + +() sub1() impure { + i = 1; +} diff --git a/crypto/func/test/i1sub2.fc b/crypto/func/test/i1sub2.fc new file mode 100644 index 00000000..0c48e192 --- /dev/null +++ b/crypto/func/test/i1sub2.fc @@ -0,0 +1,10 @@ +;; DO NOT COMPILE DIRECTLY! +;; Compile i1.fc + +#include "./i1sub1.fc"; + +() sub2() impure { + sub1(); + sub0(); + i = 2; +} diff --git a/crypto/func/test/pv.fc b/crypto/func/test/pv.fc new file mode 100644 index 00000000..d9bcd570 --- /dev/null +++ b/crypto/func/test/pv.fc @@ -0,0 +1,53 @@ +#pragma test-version-set "1.2.3"; + +;; Positive tests +#pragma version ^1.2.0; +#pragma version ^1.2.3; +#pragma version >1.2.0; +#pragma version >0.9.9; +#pragma version <1.3.0; +#pragma version <2.0.0; +#pragma version >=1.2.0; +#pragma version <=1.3.0; +#pragma version >=1.2.3; +#pragma version <=1.2.3; +#pragma version ^1.2.3; +#pragma version 1.2.3; +#pragma version =1.2.3; + +;; Negative tests +#pragma not-version ^1.1.0; +#pragma not-version ^1.0.0; +#pragma not-version ^0.2.3; +#pragma not-version ^2.2.3; +#pragma not-version ^1.3.3; +#pragma not-version >1.2.3; +#pragma not-version <1.2.3; +#pragma not-version ^1.2.4; +#pragma not-version >=1.2.4; +#pragma not-version <=1.2.2; +#pragma not-version 3.2.1; +#pragma not-version =3.2.1; + +;; Test incomplete (partial) version +#pragma version ^1.2; +#pragma version >1.2; +#pragma version <1.3; +#pragma version <2; +#pragma version >=1.2; +#pragma version <=1.3; + +;; Advanced ^ behaviour (partials) +#pragma version ^1.2; +#pragma version ^1.0; +#pragma version ^1; +#pragma version ^0; +#pragma not-version ^1.0.0; +#pragma not-version ^0.0.0; +#pragma not-version ^0.0; +#pragma not-version ^1.3; +#pragma not-version ^2; + +(int) main(int a) { + return a; +} diff --git a/crypto/func/test/s1.fc b/crypto/func/test/s1.fc new file mode 100644 index 00000000..630d0180 --- /dev/null +++ b/crypto/func/test/s1.fc @@ -0,0 +1,49 @@ +slice ascii_slice() method_id { + return "string"; +} + +slice raw_slice() method_id { + return "abcdef"s; +} + +slice addr_slice() method_id { + return "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a; +} + +int string_hex() method_id { + return "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"u; +} + +int string_minihash() method_id { + return "transfer(slice, int)"h; +} + +int string_maxihash() method_id { + return "transfer(slice, int)"H; +} + +int string_crc32() method_id { + return "transfer(slice, int)"c; +} + +builder newc() asm "NEWC"; +slice endcs(builder b) asm "ENDC" "CTOS"; +int sdeq (slice s1, slice s2) asm "SDEQ"; + +_ main() { + slice s_ascii = ascii_slice(); + slice s_raw = raw_slice(); + slice s_addr = addr_slice(); + int i_hex = string_hex(); + int i_mini = string_minihash(); + int i_maxi = string_maxihash(); + int i_crc = string_crc32(); + throw_unless(101, sdeq(s_ascii, newc().store_uint(0x737472696E67, 12 * 4).endcs())); + throw_unless(102, sdeq(s_raw, newc().store_uint(0xABCDEF, 6 * 4).endcs())); + throw_unless(103, sdeq(s_addr, newc().store_uint(4, 3).store_int(-1, 8) + .store_uint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs())); + throw_unless(104, i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435); + throw_unless(105, i_mini == 0x7a62e8a8); + throw_unless(106, i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979); + throw_unless(107, i_crc == 2235694568); +} diff --git a/crypto/parser/lexer.cpp b/crypto/parser/lexer.cpp index 87c63c3f..5c5b77d8 100644 --- a/crypto/parser/lexer.cpp +++ b/crypto/parser/lexer.cpp @@ -247,6 +247,11 @@ const Lexem& Lexer::next() { } lexem.set(std::string{src.get_ptr() + 1, end}, src.here(), qc == '`' ? Lexem::Unknown : Lexem::String); src.set_ptr(end + 1); + c = src.cur_char(); + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { + lexem.val = c; + src.set_ptr(end + 2); + } // std::cerr << lexem.name_str() << ' ' << lexem.str << std::endl; return lexem; } diff --git a/crypto/parser/symtable.h b/crypto/parser/symtable.h index 81a828a8..9489b2bc 100644 --- a/crypto/parser/symtable.h +++ b/crypto/parser/symtable.h @@ -32,7 +32,7 @@ namespace sym { typedef int var_idx_t; struct SymValBase { - enum { _Param, _Var, _Func, _Typename, _GlobVar }; + enum { _Param, _Var, _Func, _Typename, _GlobVar, _Const }; int type; int idx; SymValBase(int _type, int _idx) : type(_type), idx(_idx) { diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index f3485be9..9a273a08 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -52,30 +52,51 @@ td::Ref prepare_vm_stack(td::RefInt256 amount, td::Ref return stack_ref; } -td::Ref prepare_vm_c7(td::uint32 now, td::uint64 balance) { - // TODO: fix initialization of c7 +td::Ref prepare_vm_c7(SmartContract::Args args) { td::BitArray<256> rand_seed; rand_seed.as_slice().fill(0); td::RefInt256 rand_seed_int{true}; rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false); + + td::uint32 now = 0; + if (args.now) { + now = args.now.unwrap(); + } + + vm::CellBuilder cb; + if (args.address) { + td::BigInt256 dest_addr; + dest_addr.import_bits((*args.address).addr.as_bitslice()); + cb.store_ones(1) + .store_zeroes(2) + .store_long((*args.address).workchain, 8) + .store_int256(dest_addr, 256); + } + auto address = cb.finalize(); + auto config = td::Ref(); + + if (args.config) { + config = (*args.config)->get_root_cell(); + } + auto tuple = vm::make_tuple_ref( td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea td::make_refint(0), // actions:Integer td::make_refint(0), // msgs_sent:Integer td::make_refint(now), // unixtime:Integer - td::make_refint(0), // block_lt:Integer - td::make_refint(0), // trans_lt:Integer + td::make_refint(0), //TODO: // block_lt:Integer + td::make_refint(0), //TODO: // trans_lt:Integer std::move(rand_seed_int), // rand_seed:Integer - block::CurrencyCollection(balance).as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] - vm::load_cell_slice_ref(vm::CellBuilder().finalize()) // myself:MsgAddressInt - //vm::StackEntry::maybe(td::Ref()) + block::CurrencyCollection(args.balance).as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] + vm::load_cell_slice_ref(address), // myself:MsgAddressInt + vm::StackEntry::maybe(config) //vm::StackEntry::maybe(td::Ref()) ); // global_config:(Maybe Cell) ] = SmartContractInfo; //LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); return vm::make_tuple_ref(std::move(tuple)); } SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref stack, td::Ref c7, - vm::GasLimits gas, bool ignore_chksig) { + vm::GasLimits gas, bool ignore_chksig, td::Ref libraries) { auto gas_credit = gas.gas_credit; vm::init_op_cp0(); vm::DictionaryBase::get_empty_dictionary(); @@ -108,11 +129,15 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref SmartContract::get_init_state() const { } SmartContract::Answer SmartContract::run_method(Args args) { - td::uint32 now = 0; - if (args.now) { - now = args.now.unwrap(); - } if (!args.c7) { - args.c7 = prepare_vm_c7(now, args.balance); + args.c7 = prepare_vm_c7(args); } if (!args.limits) { bool is_internal = args.get_method_id().ok() == 0; @@ -193,28 +218,26 @@ SmartContract::Answer SmartContract::run_method(Args args) { CHECK(args.method_id); args.stack.value().write().push_smallint(args.method_id.unwrap()); auto res = - run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig); + run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig, + args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}); state_ = res.new_state; return res; } SmartContract::Answer SmartContract::run_get_method(Args args) const { - td::uint32 now = 0; - if (args.now) { - now = args.now.unwrap(); - } if (!args.c7) { - args.c7 = prepare_vm_c7(now, args.balance); + args.c7 = prepare_vm_c7(args); } if (!args.limits) { - args.limits = vm::GasLimits{1000000}; + args.limits = vm::GasLimits{1000000, 1000000}; } if (!args.stack) { args.stack = td::Ref(true); } CHECK(args.method_id); args.stack.value().write().push_smallint(args.method_id.unwrap()); - return run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig); + return run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig, + args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}); } SmartContract::Answer SmartContract::run_get_method(td::Slice method, Args args) const { diff --git a/crypto/smc-envelope/SmartContract.h b/crypto/smc-envelope/SmartContract.h index b26f4885..95dd2e84 100644 --- a/crypto/smc-envelope/SmartContract.h +++ b/crypto/smc-envelope/SmartContract.h @@ -26,6 +26,7 @@ #include "td/utils/crypto.h" #include "block/block.h" +#include "block/mc-config.h" namespace ton { class SmartContract : public td::CntObject { @@ -48,6 +49,7 @@ class SmartContract : public td::CntObject { td::Ref actions; td::int32 code; td::int64 gas_used; + td::ConstBitPtr missing_library{0}; static int output_actions_count(td::Ref list); }; @@ -61,6 +63,10 @@ class SmartContract : public td::CntObject { td::uint64 amount{0}; td::uint64 balance{0}; + td::optional address; + td::optional> config; + td::optional libraries; + Args() { } Args(std::initializer_list stack) @@ -106,6 +112,18 @@ class SmartContract : public td::CntObject { this->balance = balance; return std::move(*this); } + Args&& set_address(block::StdAddress address) { + this->address = address; + return std::move(*this); + } + Args&& set_config(std::shared_ptr& config) { + this->config = config; + return std::move(*this); + } + Args&& set_libraries(vm::Dictionary libraries) { + this->libraries = libraries; + return std::move(*this); + } td::Result get_method_id() const { if (!method_id) { diff --git a/crypto/test/modbigint.cpp b/crypto/test/modbigint.cpp new file mode 100644 index 00000000..851f0f9b --- /dev/null +++ b/crypto/test/modbigint.cpp @@ -0,0 +1,1070 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include +#include +#include +#include +#include +#include + +namespace modint { + +enum { mod_cnt = 32 }; + +// mod_cnt = 9 => integers -2^268 .. 2^268 +// mod_cnt = 18 => integers -2^537 .. 2^537 +// mod_cnt = 32 => integers -2^955 .. 2^955 +constexpr int mod[mod_cnt] = {999999937, 999999929, 999999893, 999999883, 999999797, 999999761, 999999757, 999999751, + 999999739, 999999733, 999999677, 999999667, 999999613, 999999607, 999999599, 999999587, + 999999541, 999999527, 999999503, 999999491, 999999487, 999999433, 999999391, 999999353, + 99999337, 999999323, 999999229, 999999223, 999999197, 999999193, 999999191, 999999181}; + +// invm[i][j] = mod[i]^(-1) modulo mod[j] +int invm[mod_cnt][mod_cnt]; + +int gcdx(int a, int b, int& u, int& v); + +template +struct ModArray; + +template +struct MixedRadix; + +template +struct ArrayRawDumpRef; + +template +std::ostream& raw_dump_array(std::ostream& os, const std::array& arr) { + os << '['; + for (auto x : arr) { + os << ' ' << x; + } + return os << " ]"; +} + +template +struct MixedRadix { + enum { n = N }; + int a[N]; + MixedRadix(int v) { + set_int(v); + } + MixedRadix() = default; + MixedRadix(const MixedRadix&) = default; + MixedRadix(std::initializer_list l) { + auto sz = std::min(l.size(), (std::size_t)N); + std::copy(l.begin(), l.begin() + sz, a); + std::fill(a + sz, a + N, 0); + } + MixedRadix(const std::array& arr) { + std::copy(arr.begin(), arr.end(), a); + } + template + MixedRadix(const MixedRadix& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + } + MixedRadix(const ModArray& other); + MixedRadix(const ModArray& other, bool sgnd); + + MixedRadix& set_zero() { + std::fill(a, a + N, 0); + return *this; + } + MixedRadix& set_one() { + a[0] = 1; + std::fill(a + 1, a + N, 0); + return *this; + } + MixedRadix& set_int(int v) { + a[0] = v; + std::fill(a + 1, a + N, 0); + return *this; + } + + MixedRadix copy() const { + return MixedRadix{*this}; + } + + static const int* mod_array() { + return mod; + } + + static int modulus(int i) { + return mod[i]; + } + + int sgn() const { + int i = N - 1; + while (i >= 0 && !a[i]) { + --i; + } + return i < 0 ? 0 : (a[i] > 0 ? 1 : -1); + } + + int cmp(const MixedRadix& other) const { + int i = N - 1; + while (i >= 0 && a[i] == other.a[i]) { + --i; + } + return i < 0 ? 0 : (a[i] > other.a[i] ? 1 : -1); + } + + bool is_small() const { + return !a[N - 1] || a[N - 1] == -1; + } + + bool operator==(const MixedRadix& other) const { + return std::equal(a, a + N, other.a); + } + + bool operator!=(const MixedRadix& other) const { + return !std::equal(a, a + N, other.a); + } + + bool operator<(const MixedRadix& other) const { + return cmp(other) < 0; + } + + bool operator<=(const MixedRadix& other) const { + return cmp(other) <= 0; + } + + bool operator>(const MixedRadix& other) const { + return cmp(other) > 0; + } + + bool operator>=(const MixedRadix& other) const { + return cmp(other) >= 0; + } + + explicit operator bool() const { + return sgn(); + } + + bool operator!() const { + return !sgn(); + } + + MixedRadix& negate() { + int i = 0; + while (i < N - 1 && !a[i]) { + i++; + } + a[i]--; + for (; i < N; i++) { + a[i] = mod[i] - a[i] - 1; + } + a[N - 1] -= mod[N - 1]; + return *this; + } + + static const MixedRadix& pow2(int power); + static MixedRadix negpow2(int power) { + return -pow2(power); + } + + template + const MixedRadix& as_shorter() const { + static_assert(M <= N); + return *reinterpret_cast*>(this); + } + + MixedRadix& import_mod_array(const int* data, bool sgnd = true) { + for (int i = 0; i < N; i++) { + a[i] = data[i] % mod[i]; + } + for (int i = 0; i < N; i++) { + if (a[i] < 0) { + a[i] += mod[i]; + } + for (int j = i + 1; j < N; j++) { + a[j] = (int)((long long)(a[j] - a[i]) * invm[i][j] % mod[j]); + } + } + if (sgnd && a[N - 1] > (mod[N - 1] >> 1)) { + a[N - 1] -= mod[N - 1]; + } + return *this; + } + + MixedRadix& operator=(const MixedRadix&) = default; + + template + MixedRadix& operator=(const MixedRadix& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + } + + MixedRadix& import_mod_array(const ModArray& other, bool sgnd = true); + + MixedRadix& operator=(const ModArray& other) { + return import_mod_array(other); + } + + MixedRadix& set_sum(const MixedRadix& x, const MixedRadix& y, int factor = 1) { + long long carry = 0; + for (int i = 0; i < N; i++) { + long long acc = x.a[i] + carry + (long long)factor * y.a[i]; + carry = acc / mod[i]; + a[i] = (int)(acc - carry * mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + --carry; + } + } + if (a[N - 1] >= 0 && carry == -1) { + a[N - 1] -= mod[N - 1]; + } + return *this; + } + + MixedRadix& operator+=(const MixedRadix& other) { + return set_sum(*this, other); + } + + MixedRadix& operator-=(const MixedRadix& other) { + return set_sum(*this, other, -1); + } + + static const MixedRadix& zero(); + static const MixedRadix& one(); + + MixedRadix& operator*=(int factor) { + return set_sum(zero(), *this, factor); + } + + MixedRadix operator-() const { + MixedRadix copy{*this}; + copy.negate(); + return copy; + } + + MixedRadix operator+(const MixedRadix& other) const { + MixedRadix res; + res.set_sum(*this, other); + return res; + } + + MixedRadix operator-(const MixedRadix& other) const { + MixedRadix res; + res.set_sum(*this, other, -1); + return res; + } + + MixedRadix operator*(int factor) const { + MixedRadix res; + res.set_sum(zero(), *this, factor); + return res; + } + + int operator%(int b) const { + int x = a[N - 1] % b; + for (int i = N - 2; i >= 0; --i) { + x = ((long long)x * mod[i] + a[i]) % b; + } + return ((x ^ b) < 0 && x) ? x + b : x; + } + + explicit operator double() const { + double acc = 0.; + for (int i = N - 1; i >= 0; --i) { + acc = acc * mod[i] + a[i]; + } + return acc; + } + + explicit operator long long() const { + long long acc = 0.; + for (int i = N - 1; i >= 0; --i) { + acc = acc * mod[i] + a[i]; + } + return acc; + } + + MixedRadix& to_base(int base) { + int k = N - 1; + while (k > 0 && !a[k]) { + --k; + } + if (k <= 0) { + return *this; + } + for (int i = k - 1; i >= 0; --i) { + // a[i..k] := a[i+1..k] * mod[i] + a[i] + long long carry = a[i]; + for (int j = i; j < k; j++) { + long long t = (long long)a[j + 1] * mod[i] + carry; + carry = t / base; + a[j] = (int)(t - carry * base); + } + a[k] = (int)carry; + } + return *this; + } + + std::ostream& print_dec_destroy(std::ostream& os) { + int s = sgn(); + if (s < 0) { + os << '-'; + negate(); + } else if (!s) { + os << '0'; + return os; + } + to_base(1000000000); + int i = N - 1; + while (!a[i] && i > 0) { + --i; + } + os << a[i]; + while (--i >= 0) { + char buff[12]; + sprintf(buff, "%09d", a[i]); + os << buff; + } + return os; + } + + std::ostream& print_dec(std::ostream& os) const& { + MixedRadix copy{*this}; + return copy.print_dec_destroy(os); + } + + std::ostream& print_dec(std::ostream& os) && { + return print_dec_destroy(os); + } + + std::string to_dec_string_destroy() { + std::ostringstream os; + print_dec_destroy(os); + return std::move(os).str(); + } + + std::string to_dec_string() const& { + MixedRadix copy{*this}; + return copy.to_dec_string_destroy(); + } + + std::string to_dec_string() && { + return to_dec_string_destroy(); + } + + bool to_binary_destroy(unsigned char* arr, int size, bool sgnd = true) { + if (size <= 0) { + return false; + } + int s = (sgnd ? sgn() : 1); + memset(arr, 0, size); + if (s < 0) { + negate(); + } else if (!s) { + return true; + } + to_base(1 << 30); + long long acc = 0; + int bits = 0, j = size; + for (int i = 0; i < N; i++) { + if (!j && a[i]) { + return false; + } + acc += ((long long)a[i] << bits); + bits += 30; + while (bits >= 8 && j > 0) { + arr[--j] = (unsigned char)(acc & 0xff); + bits -= 8; + acc >>= 8; + } + } + while (j > 0) { + arr[--j] = (unsigned char)(acc & 0xff); + acc >>= 8; + } + if (acc) { + return false; + } + if (!sgnd) { + return true; + } + if (s >= 0) { + return arr[0] <= 0x7f; + } + j = size - 1; + while (j >= 0 && !arr[j]) { + --j; + } + assert(j >= 0); + arr[j] = (unsigned char)(-arr[j]); + while (--j >= 0) { + arr[j] = (unsigned char)~arr[j]; + } + return arr[0] >= 0x80; + } + + bool to_binary(unsigned char* arr, int size, bool sgnd = true) const& { + MixedRadix copy{*this}; + return copy.to_binary_destroy(arr, size, sgnd); + } + + bool to_binary(unsigned char* arr, int size, bool sgnd = true) && { + return to_binary_destroy(arr, size, sgnd); + } + + std::ostream& raw_dump(std::ostream& os) const { + return raw_dump_array(os, a); + } + + ArrayRawDumpRef dump() const { + return {a}; + } +}; + +template +struct ModArray { + enum { n = N }; + int a[N]; + ModArray(int v) { + set_int(v); + } + ModArray(long long v) { + set_long(v); + } + ModArray(long v) { + set_long(v); + } + ModArray() = default; + ModArray(const ModArray&) = default; + ModArray(std::initializer_list l) { + auto sz = std::min(l.size(), (std::size_t)N); + std::copy(l.begin(), l.begin() + sz, a); + std::fill(a + sz, a + N, 0); + } + ModArray(const std::array& arr) { + std::copy(arr.begin(), arr.end(), a); + } + template + ModArray(const ModArray& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + } + ModArray(const int* p) : a(p) { + } + ModArray(std::string str) { + assert(from_dec_string(str) && "not a decimal number"); + } + + ModArray& set_zero() { + std::fill(a, a + N, 0); + return *this; + } + ModArray& set_one() { + std::fill(a, a + N, 1); + return *this; + } + + ModArray& set_int(int v) { + if (v >= 0) { + std::fill(a, a + N, v); + } else { + for (int i = 0; i < N; i++) { + a[i] = mod[i] + v; + } + } + return *this; + } + + ModArray& set_long(long long v) { + for (int i = 0; i < N; i++) { + a[i] = v % mod[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray copy() const { + return ModArray{*this}; + } + + static const int* mod_array() { + return mod; + } + + static int modulus(int i) { + return mod[i]; + } + + static const ModArray& zero(); + static const ModArray& one(); + + ModArray& operator=(const ModArray&) = default; + + template + ModArray& operator=(const ModArray& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + return *this; + } + + ModArray& negate() { + for (int i = 0; i < N; i++) { + a[i] = (a[i] ? mod[i] - a[i] : 0); + } + return *this; + } + + ModArray& norm_neg() { + for (int i = 0; i < N; i++) { + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& normalize() { + for (int i = 0; i < N; i++) { + a[i] %= mod[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + bool is_zero() const { + for (int i = 0; i < N; i++) { + if (a[i]) { + return false; + } + } + return true; + } + + explicit operator bool() const { + return !is_zero(); + } + + bool operator!() const { + return is_zero(); + } + + bool operator==(const ModArray& other) const { + return std::equal(a, a + N, other.a); + } + + bool operator!=(const ModArray& other) const { + return !std::equal(a, a + N, other.a); + } + + bool operator==(long long val) const { + for (int i = 0; i < N; i++) { + int r = (int)(val % mod[i]); + if (a[i] != (r < 0 ? r + mod[i] : r)) { + return false; + } + } + return true; + } + + bool operator!=(long long val) const { + return !operator==(val); + } + + long long try_get_long() const { + return (long long)(MixedRadix<3>(*this)); + } + + bool fits_long() const { + return operator==(try_get_long()); + } + + explicit operator long long() const { + auto v = try_get_long(); + return operator==(v) ? v : -0x8000000000000000; + } + + ModArray& set_sum(const ModArray& x, const ModArray& y) { + for (int i = 0; i < N; i++) { + a[i] = x.a[i] + y.a[i]; + if (a[i] >= mod[i]) { + a[i] -= mod[i]; + } + } + return *this; + } + + ModArray& operator+=(const ModArray& other) { + for (int i = 0; i < N; i++) { + a[i] += other.a[i]; + if (a[i] >= mod[i]) { + a[i] -= mod[i]; + } + } + return *this; + } + + ModArray& operator+=(long long v) { + for (int i = 0; i < N; i++) { + a[i] = (int)((a[i] + v) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& operator-=(const ModArray& other) { + for (int i = 0; i < N; i++) { + a[i] -= other.a[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& operator-=(long long v) { + return (operator+=)(-v); + } + + ModArray& mul_arr(const int other[]) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * other[i]) % mod[i]); + } + return *this; + } + + ModArray& operator*=(const ModArray& other) { + return mul_arr(other.a); + } + + template + ModArray& operator*=(const ModArray& other) { + static_assert(M >= N); + return mul_arr(other.a); + } + + ModArray& operator*=(int v) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * v) % mod[i]); + } + return (v >= 0 ? *this : norm_neg()); + } + + ModArray& operator*=(long long v) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * (v % mod[i])) % mod[i]); + } + return (v >= 0 ? *this : norm_neg()); + } + + ModArray& mul_add(int v, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * v + w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + // *this = (*this * other) + w + ModArray& mul_add(const ModArray& other, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * other.a[i] + w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + // *this = (*this << shift) + w + ModArray& lshift_add(int shift, long long w) { + return mul_add(pow2(shift), w); + } + + // *this = *this + other * w + ModArray& add_mul(const ModArray& other, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)((a[i] + other.a[i] * w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + // *this += w << shift + ModArray& add_lshift(int shift, long long w) { + return add_mul(pow2(shift), w); + } + + ModArray operator+(const ModArray& other) const { + ModArray copy{*this}; + copy += other; + return copy; + } + + ModArray operator-(const ModArray& other) const { + ModArray copy{*this}; + copy -= other; + return copy; + } + + ModArray operator+(long long other) const { + ModArray copy{*this}; + copy += other; + return copy; + } + + ModArray operator-(long long other) const { + ModArray copy{*this}; + copy += -other; + return copy; + } + + ModArray operator-() const { + ModArray copy{*this}; + copy.negate(); + return copy; + } + + ModArray operator*(const ModArray& other) const { + ModArray copy{*this}; + copy *= other; + return copy; + } + + ModArray operator*(long long other) const { + ModArray copy{*this}; + copy *= other; + return copy; + } + + bool invert() { + for (int i = 0; i < N; i++) { + int t; + if (gcdx(a[i], mod[i], a[i], t) != 1) { + return false; + } + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return true; + } + + bool try_divide(const ModArray& other) { + for (int i = 0; i < N; i++) { + int q, t; + if (gcdx(other.a[i], mod[i], q, t) != 1) { + return false; + } + a[i] = (int)((long long)a[i] * q % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return true; + } + + ModArray& operator/=(const ModArray& other) { + assert(try_divide(other) && "division by zero?"); + return *this; + } + + ModArray operator/(const ModArray& other) { + ModArray copy{*this}; + copy /= other; + return copy; + } + + static const ModArray& pow2(int power); + static const ModArray& negpow2(int power); + + ModArray& operator<<=(int lshift) { + return operator*=(pow2(lshift)); + } + + ModArray operator<<(int lshift) const { + return operator*(pow2(lshift)); + } + + ModArray& operator>>=(int rshift) { + return operator/=(pow2(rshift)); + } + + ModArray operator>>(int rshift) const { + return operator/(pow2(rshift)); + } + + template + const ModArray& as_shorter() const { + static_assert(M <= N); + return *reinterpret_cast*>(this); + } + + MixedRadix& to_mixed_radix(MixedRadix& dest, bool sgnd = true) const { + return dest.import_mod_array(a, sgnd); + } + + MixedRadix to_mixed_radix(bool sgnd = true) const { + return MixedRadix(*this, sgnd); + } + + int operator%(int div) const { + return to_mixed_radix() % div; + } + + explicit operator double() const { + return (double)to_mixed_radix(); + } + + std::string to_dec_string() const { + return MixedRadix(*this).to_dec_string(); + } + + std::ostream& print_dec(std::ostream& os, bool sgnd = true) const { + return MixedRadix(*this, sgnd).print_dec(os); + } + + bool to_binary(unsigned char* arr, int size, bool sgnd = true) const { + return MixedRadix(*this, sgnd).to_binary(arr, size, sgnd); + } + + template + bool to_binary(std::array& arr, bool sgnd = true) const { + return to_binary(arr.data(), M, sgnd); + } + + bool from_dec_string(const char* start, const char* end) { + set_zero(); + if (start >= end) { + return false; + } + bool sgn = (*start == '-'); + if (sgn && ++start == end) { + return false; + } + int acc = 0, pow = 1; + while (start < end) { + if (*start < '0' || *start > '9') { + return false; + } + acc = acc * 10 + (*start++ - '0'); + pow *= 10; + if (pow >= 1000000000) { + mul_add(pow, acc); + pow = 1; + acc = 0; + } + } + if (pow > 1) { + mul_add(pow, acc); + } + if (sgn) { + negate(); + } + return true; + } + + bool from_dec_string(std::string str) { + return from_dec_string(str.data(), str.data() + str.size()); + } + + ModArray& from_binary(const unsigned char* arr, int size, bool sgnd = true) { + set_zero(); + if (size <= 0) { + return *this; + } + int i = 0, pow = 0; + long long acc = (sgnd && arr[0] >= 0x80 ? -1 : 0); + while (i < size && arr[i] == (unsigned char)acc) { + i++; + } + for (; i < size; i++) { + pow += 8; + acc = (acc << 8) + arr[i]; + if (pow >= 56) { + lshift_add(pow, acc); + acc = pow = 0; + } + } + if (pow || acc) { + lshift_add(pow, acc); + } + return *this; + } + + template + ModArray& from_binary(const std::array& arr, bool sgnd = true) { + return from_binary(arr.data(), M, sgnd); + } + + std::ostream& raw_dump(std::ostream& os) const { + return raw_dump_array(os, a); + } + + ArrayRawDumpRef dump() const { + return {a}; + } +}; + +template +MixedRadix::MixedRadix(const ModArray& other) { + import_mod_array(other.a); +} + +template +MixedRadix::MixedRadix(const ModArray& other, bool sgnd) { + import_mod_array(other.a, sgnd); +} + +template +MixedRadix& MixedRadix::import_mod_array(const ModArray& other, bool sgnd) { + return import_mod_array(other.a, sgnd); +} + +template +std::ostream& operator<<(std::ostream& os, const ModArray& x) { + return x.print_dec(os); +} + +template +std::ostream& operator<<(std::ostream& os, const MixedRadix& x) { + return x.print_dec(os); +} + +template +std::ostream& operator<<(std::ostream& os, MixedRadix&& x) { + return x.print_dec_destroy(os); +} + +template +struct ArrayRawDumpRef { + const std::array& ref; + ArrayRawDumpRef(const std::array& _ref) : ref(_ref){}; +}; + +template +std::ostream& operator<<(std::ostream& os, ArrayRawDumpRef rd_ref) { + return raw_dump_array(os, rd_ref.ref); +}; + +constexpr int pow2_cnt = 1001; + +ModArray Zero(0), One(1), Pow2[pow2_cnt], NegPow2[pow2_cnt]; +MixedRadix Zero_mr(0), One_mr(1), Pow2_mr[pow2_cnt], NegPow2_mr[pow2_cnt]; + +template +const MixedRadix& MixedRadix::pow2(int power) { + return Pow2_mr[power].as_shorter(); +} + +/* +template +const MixedRadix& MixedRadix::negpow2(int power) { + return NegPow2_mr[power].as_shorter(); +} +*/ + +template +const ModArray& ModArray::pow2(int power) { + return Pow2[power].as_shorter(); +} + +template +const ModArray& ModArray::negpow2(int power) { + return NegPow2[power].as_shorter(); +} + +template +const ModArray& ModArray::zero() { + return Zero.as_shorter(); +} + +template +const ModArray& ModArray::one() { + return One.as_shorter(); +} + +template +const MixedRadix& MixedRadix::zero() { + return Zero_mr.as_shorter(); +} + +template +const MixedRadix& MixedRadix::one() { + return One_mr.as_shorter(); +} + +void init_pow2() { + Pow2[0].set_one(); + Pow2_mr[0].set_one(); + for (int i = 1; i < pow2_cnt; i++) { + Pow2[i].set_sum(Pow2[i - 1], Pow2[i - 1]); + Pow2_mr[i].set_sum(Pow2_mr[i - 1], Pow2_mr[i - 1]); + } + for (int i = 0; i < pow2_cnt; i++) { + NegPow2[i] = -Pow2[i]; + NegPow2_mr[i] = -Pow2_mr[i]; + } +} + +int gcdx(int a, int b, int& u, int& v) { + int a1 = 1, a2 = 0, b1 = 0, b2 = 1; + while (b) { + int q = a / b; + int t = a - q * b; + a = b; + b = t; + t = a1 - q * b1; + a1 = b1; + b1 = t; + t = a2 - q * b2; + a2 = b2; + b2 = t; + } + u = a1; + v = a2; + return a; +} + +void init_invm() { + for (int i = 0; i < mod_cnt; i++) { + assert(mod[i] > 0 && mod[i] <= (1 << 30)); + for (int j = 0; j < i; j++) { + assert(gcdx(mod[i], mod[j], invm[i][j], invm[j][i]) == 1); + if (invm[i][j] < 0) { + invm[i][j] += mod[j]; + } + if (invm[j][i] < 0) { + invm[j][i] += mod[i]; + } + } + } +} + +void init() { + init_invm(); + init_pow2(); +} + +} // namespace modint diff --git a/crypto/test/test-bigint.cpp b/crypto/test/test-bigint.cpp new file mode 100644 index 00000000..bf85e2ad --- /dev/null +++ b/crypto/test/test-bigint.cpp @@ -0,0 +1,876 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include "common/refcnt.hpp" +#include "common/bigint.hpp" +#include "common/refint.h" +#include "modbigint.cpp" + +#include "td/utils/tests.h" + +int mkint_chk_mode = -1, res_chk_mode = 0; +long long iterations = 100000, cur_iteration = -1, debug_iteration = -2; +#define IFDEBUG if (cur_iteration == debug_iteration || debug_iteration == -3) + +using BInt = modint::ModArray<18>; // integers up to 2^537 +using MRInt = modint::MixedRadix<18>; // auxiliary integer representation for printing, comparing etc + +MRInt p2_256, np2_256, p2_63, np2_63; +constexpr long long ll_min = -2 * (1LL << 62), ll_max = ~ll_min; +constexpr double dbl_pow256 = 1.1579208923731619542e77 /* 0x1p256 */; // 2^256 + +std::mt19937_64 Random(666); + +template +bool equal(td::RefInt256 x, T y) { + return !td::cmp(x, y); +} + +bool equal_or_nan(td::RefInt256 x, td::RefInt256 y) { + return equal(x, y) || (!x->is_valid() && !y->fits_bits(257)) || (!y->is_valid() && !x->fits_bits(257)); +} + +#define CHECK_EQ(__x, __y) CHECK(equal(__x, __y)) +#define CHECK_EQ_NAN(__x, __y) CHECK(equal_or_nan(__x, __y)) + +bool mr_in_range(const MRInt& x) { + return x < p2_256 && x >= np2_256; +} + +bool mr_is_small(const MRInt& x) { + return x < p2_63 && x >= np2_63; +} + +bool mr_fits_bits(const MRInt& x, int bits) { + if (bits > 0) { + return x < MRInt::pow2(bits - 1) && x >= MRInt::negpow2(bits - 1); + } else { + return !bits && !x.sgn(); + } +} + +bool mr_ufits_bits(const MRInt& x, int bits) { + return bits >= 0 && x.sgn() >= 0 && x < MRInt::pow2(bits); +} + +struct ShowBin { + unsigned char* data; + ShowBin(unsigned char _data[64]) : data(_data) { + } +}; + +std::ostream& operator<<(std::ostream& os, ShowBin bin) { + int i = 0, s = bin.data[0]; + if (s == 0 || s == 0xff) { + while (i < 64 && bin.data[i] == s) { + i++; + } + } + if (i >= 3) { + os << (s ? "ff..ff" : "00..00"); + } else { + i = 0; + } + constexpr static char hex_digits[] = "0123456789abcdef"; + while (i < 64) { + int t = bin.data[i++]; + os << hex_digits[t >> 4] << hex_digits[t & 15]; + } + return os; +} + +std::ostream& operator<<(std::ostream& os, const td::AnyIntView& x) { + os << '['; + for (int i = 0; i < x.size(); i++) { + os << ' ' << x.digits[i]; + } + os << " ]"; + return os; +} + +template +bool extract_value_any_bool(BInt& val, const td::AnyIntView& x, bool chk_norm = true) { + int n = x.size(); + if (n <= 0 || n > x.max_size() || (!x.digits[n - 1] && n > 1)) { + return false; + } + assert(n == 1 || x.digits[n - 1] != 0); + val.set_zero(); + for (int i = n - 1; i >= 0; --i) { + val.lshift_add(T::word_shift, x.digits[i]); + if (chk_norm && (x.digits[i] < -T::Half || x.digits[i] >= T::Half)) { + return false; // unnormalized + } + } + return true; +} + +template +bool extract_value_bool(BInt& val, const T& x, bool chk_norm = true) { + return extract_value_any_bool(val, x.as_any_int(), chk_norm); +} + +BInt extract_value_any(const td::AnyIntView& x, bool chk_norm = true) { + BInt res; + CHECK(extract_value_any_bool(res, x, chk_norm)); + return res; +} + +template +BInt extract_value(const T& x, bool chk_norm = true) { + return extract_value_any(x.as_any_int(), chk_norm); +} + +template +BInt extract_value_alt(const T& x) { + BInt res; + const int* md = res.mod_array(); + for (int i = 0; i < res.n / 2; i++) { + T copy{x}; + int m1 = md[2 * i], m2 = md[2 * i + 1]; + long long rem = copy.divmod_short((long long)m1 * m2); + res.a[2 * i] = (int)(rem % m1); + res.a[2 * i + 1] = (int)(rem % m2); + } + if (res.n & 1) { + T copy{x}; + res.a[res.n - 1] = (int)copy.divmod_short(md[res.n - 1]); + } + return res; +} + +constexpr int min_spec_int = -0xfd08, max_spec_int = 0xfd07; +// x = sgn*(ord*256+a*16+b) => sgn*((32+a)*2^(ord-2) + b - 8) +// x = -0xfd08 => -2^256 ... x = 0xfd07 => 2^256 - 1 +td::RefInt256 make_special_int(int x, BInt* ptr = nullptr, unsigned char bin[64] = nullptr) { + bool sgn = (x < 0); + if (sgn) { + x = -x; + } + int ord = (x >> 8) - 2, a = 32 + ((x >> 4) & 15), b = (x & 15) - 8; + if (ord < 0) { + a >>= -ord; + ord = 0; + } + if (sgn) { + a = -a; + b = -b; + } + if (ptr) { + ptr->set_int(a); + *ptr <<= ord; + *ptr += b; + } + if (bin) { + int acc = b, r = ord; + for (int i = 63; i >= 0; --i) { + if (r < 8) { + acc += (a << r); + r = 1024; + } + r -= 8; + bin[i] = (unsigned char)(acc & 0xff); + acc >>= 8; + } + } + return (td::make_refint(a) << ord) + b; +} + +int rand_int(int min, int max) { + return min + (int)(Random() % (max - min + 1)); +} + +unsigned randu() { + return (unsigned)(Random() << 16); +} + +bool coin() { + return Random() & (1 << 28); +} + +// returns 0 with probability 1/2, 1 with prob. 1/4, ..., k with prob. 1/2^(k+1) +int randexp(int max = 63, int min = 0) { + return min + __builtin_clzll(Random() | (1ULL << (63 - max + min))); +} + +void bin_add_small(unsigned char bin[64], long long val, int shift = 0) { + val <<= shift & 7; + for (int i = 63 - (shift >> 3); i >= 0 && val; --i) { + val += bin[i]; + bin[i] = (unsigned char)val; + val >>= 8; + } +} + +// adds sgn * (random number less than 2^(ord - ord2)) * 2^ord2 +td::RefInt256 add_random_bits(td::RefInt256 x, BInt& val, unsigned char bin[64], int ord2, int ord, int sgn = 1) { + int t; + do { + t = std::max((ord - 1) & -16, ord2); + int a = sgn * rand_int(0, (1 << (ord - t)) - 1); + // add a << t + val.add_lshift(t, a); + x += td::make_refint(a) << t; + bin_add_small(bin, a, t); + ord = t; + } while (t > ord2); + return x; +} + +// generates a random integer in range -2^256 .. 2^256-1 (and sometimes outside) +// distribution is skewed towards +/- 2^n +/- 2^n +/- smallint, but completely random integers are also generated +td::RefInt256 make_random_int0(BInt& val, unsigned char bin[64]) { + memset(bin, 0, 64); + int ord = rand_int(-257, 257); + if (ord <= 2 && ord >= -2) { + // -2..2 represent themselves + val.set_int(ord); + bin_add_small(bin, ord); + return td::make_refint(ord); + } + int sgn = (ord < 0 ? -1 : 1); + ord = sgn * ord - 1; + int f = std::min(ord, randexp(15)), a = sgn * rand_int(1 << f, (2 << f) - 1); + ord -= f; + // first summand is a << ord + auto res = td::make_refint(a) << ord; + val.set_int(a); + val <<= ord; + bin_add_small(bin, a, ord); + if (!ord) { + // all bits ready + return res; + } + for (int s = 0; s < 2 && ord; s++) { + // decide whether we want an intermediate order (50%), and whether we want randomness above/below that order + int ord2 = (s ? 0 : std::max(0, rand_int(~ord, ord - 1))); + if (!rand_int(0, 4)) { // 20% + // random bits between ord2 and ord + res = add_random_bits(std::move(res), val, bin, ord2, ord, sgn); + } + if (rand_int(0, 4)) { // 80% + // non-zero adjustment + f = randexp(15); + a = rand_int(-(2 << f) + 1, (2 << f) - 1); + ord = std::max(ord2 - f, 0); + // add a << ord + val.add_lshift(ord, a); + res += (td::make_refint(a) << ord); + bin_add_small(bin, a, ord); + } + } + return res; +} + +td::RefInt256 make_random_int(BInt& val, unsigned char bin[64]) { + while (true) { + auto res = make_random_int0(val, bin); + if (res->fits_bits(257)) { + return res; + } + } +} + +void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt* valptr = nullptr, + const unsigned char bin[64] = nullptr) { + CHECK(x.not_null() && (in_range <= -2 || x->is_valid())); + if (!x->is_valid()) { + // not much to check when x is a NaN + unsigned char bytes[64]; + if (valptr) { + // check that the true answer at `valptr` is out of range + CHECK(!mr_in_range(valptr->to_mixed_radix())); + if (mode & 0x200) { + // check BInt binary export + valptr->to_binary(bytes, 64); + if (bin) { + // check that the two true answers match + CHECK(!memcmp(bin, bytes, 64)); + } else { + bin = bytes; + } + } + } + if (bin) { + // check that the true answer in `bin` is out of range + int i = 0, sgn = (bin[0] >= 0x80 ? -1 : 0); + while (i < 32 && bin[i] == (unsigned char)sgn) + ; + CHECK(i < 32); + if (valptr && (mode & 0x100)) { + // check BInt binary export + BInt val2; + val2.from_binary(bin, 64); + CHECK(*valptr == val2); + } + } + return; + } + unsigned char bytes[64]; + CHECK(x->export_bytes(bytes, 64)); + if (bin) { + CHECK(!memcmp(bytes, bin, 64)); + } + BInt val = extract_value(*x); + if (valptr) { + if (val != *valptr) { + std::cerr << "extracted " << val << " from " << x << ' ' << x->as_any_int() << ", expected " << *valptr + << std::endl; + } + CHECK(val == *valptr); + } + if (mode & 1) { + BInt val2 = extract_value_alt(*x); + CHECK(val == val2); + } + if (mode & 2) { + // check binary import + td::BigInt256 y; + y.import_bytes(bytes, 64); + CHECK(y == *x); + } + if (mode & 0x100) { + // check binary import for BInt + BInt val2; + val2.from_binary(bytes, 64); + CHECK(val == val2); + } + // check if small (fits into 64 bits) + long long xval = (long long)val; + bool is_small = (xval != ll_min || val == xval); + CHECK(is_small == x->fits_bits(64)); + if (is_small) { + // special check for small (64-bit) values + CHECK(x->to_long() == xval); + CHECK((long long)__builtin_bswap64(*(long long*)(bytes + 64 - 8)) == xval); + CHECK(in_range); + // check sign + CHECK(x->sgn() == (xval > 0 ? 1 : (xval < 0 ? -1 : 0))); + // check comparison with long long + CHECK(x == xval); + CHECK(!cmp(x, xval)); + if (mode & 4) { + // check constructor from long long + CHECK(!cmp(x, td::make_refint(xval))); + if (xval != ll_min) { + CHECK(x > xval - 1); + CHECK(x > td::make_refint(xval - 1)); + } + if (xval != ll_max) { + CHECK(x < xval + 1); + CHECK(x < td::make_refint(xval + 1)); + } + } + if (!(mode & ~0x107)) { + return; // fast check for small ints in this case + } + } + + MRInt mval(val); // somewhat slow + bool val_in_range = mr_in_range(mval); + CHECK(x->fits_bits(257) == val_in_range); + if (in_range >= 0) { + CHECK((int)val_in_range == in_range); + } + if (mode & 0x200) { + // check binary export for BInt + unsigned char bytes2[64]; + mval.to_binary(bytes2, 64); + CHECK(!memcmp(bytes, bytes2, 64)); + } + // check sign + int sgn = mval.sgn(); + CHECK(x->sgn() == sgn); + CHECK(is_small == mr_is_small(mval)); + if (is_small) { + CHECK((long long)mval == xval); + } + if (mode & 0x10) { + // check decimal export + std::string dec = mval.to_dec_string(); + CHECK(x->to_dec_string() == dec); + // check decimal import + td::BigInt256 y; + int l = y.parse_dec(dec); + CHECK((std::size_t)l == dec.size() && y == *x); + if (mode & 0x1000) { + // check decimal import for BInt + BInt val2; + CHECK(val2.from_dec_string(dec) && val2 == val); + } + } + if (mode & 0x20) { + // check binary bit size + int sz = x->bit_size(); + CHECK(sz >= 0 && sz <= 300); + CHECK(x->fits_bits(sz) && (!sz || !x->fits_bits(sz - 1))); + CHECK(mr_fits_bits(mval, sz) && !mr_fits_bits(mval, sz - 1)); + int usz = x->bit_size(false); + CHECK(sgn >= 0 || usz == 0x7fffffff); + if (sgn >= 0) { + CHECK(x->unsigned_fits_bits(usz) && (!usz || !x->unsigned_fits_bits(usz - 1))); + CHECK(mr_ufits_bits(mval, usz) && !mr_ufits_bits(mval, usz - 1)); + } else { + CHECK(!x->unsigned_fits_bits(256) && !x->unsigned_fits_bits(300)); + } + } +} + +void init_aux() { + np2_256 = p2_256 = MRInt::pow2(256); + np2_256.negate(); + CHECK(np2_256 == MRInt::negpow2(256)); + p2_63 = np2_63 = MRInt::pow2(63); + np2_63.negate(); + CHECK(np2_63 == MRInt::negpow2(63)); +} + +std::vector SpecInt; +BInt SpecIntB[max_spec_int - min_spec_int + 1]; + +void init_check_special_ints() { + std::cerr << "check special ints" << std::endl; + BInt b; + unsigned char binary[64]; + for (int idx = min_spec_int - 512; idx <= max_spec_int + 512; idx++) { + td::RefInt256 x = make_special_int(idx, &b, binary); + check_one_int_repr(x, mkint_chk_mode, idx >= min_spec_int && idx <= max_spec_int, &b, binary); + if (idx >= min_spec_int && idx <= max_spec_int) { + SpecIntB[idx - min_spec_int] = b; + SpecInt.push_back(std::move(x)); + } + } +} + +void check_res(td::RefInt256 y, const BInt& yv) { + check_one_int_repr(std::move(y), res_chk_mode, -2, &yv); +} + +void check_unary_ops_on(td::RefInt256 x, const BInt& xv) { + // NEGATE + BInt yv = -xv; + check_res(-x, yv); + // NOT + check_res(~x, yv -= 1); +} + +void check_unary_ops() { + std::cerr << "check unary ops" << std::endl; + for (int idx = min_spec_int; idx <= max_spec_int; idx++) { + check_unary_ops_on(SpecInt[idx - min_spec_int], SpecIntB[idx - min_spec_int]); + } +} + +void check_pow2_ops(int shift) { + // POW2 + td::RefInt256 r{true}; + r.unique_write().set_pow2(shift); + check_res(r, BInt::pow2(shift)); + // POW2DEC + r.unique_write().set_pow2(shift).add_tiny(-1).normalize(); + check_res(r, BInt::pow2(shift) - 1); + // NEGPOW2 + r.unique_write().set_pow2(shift).negate().normalize(); + check_res(r, -BInt::pow2(shift)); +} + +void check_pow2_ops() { + std::cerr << "check power-2 ops" << std::endl; + for (int i = 0; i <= 256; i++) { + check_pow2_ops(i); + } +} + +void check_shift_ops_on(int shift, td::RefInt256 x, const BInt& xv, const MRInt& mval) { + // LSHIFT + check_res(x << shift, xv << shift); + // FITS + CHECK(x->fits_bits(shift) == mr_fits_bits(mval, shift)); + // UFITS + CHECK(x->unsigned_fits_bits(shift) == mr_ufits_bits(mval, shift)); + // ADDPOW2 / SUBPOW2 + auto y = x; + y.write().add_pow2(shift).normalize(); + check_res(std::move(y), xv + BInt::pow2(shift)); + y = x; + y.write().sub_pow2(shift).normalize(); + check_res(std::move(y), xv - BInt::pow2(shift)); + // RSHIFT, MODPOW2 + for (int round_mode = -1; round_mode <= 1; round_mode++) { + auto r = x, q = td::rshift(x, shift, round_mode); // RSHIFT + CHECK(q.not_null() && q->is_valid()); + r.write().mod_pow2(shift, round_mode).normalize(); // MODPOW2 + CHECK(r.not_null() && r->is_valid()); + if (round_mode < 0) { + CHECK(!cmp(x >> shift, q)); // operator>> should be equivalent to td::rshift + } + BInt qv = extract_value(*q), rv = extract_value(*r); + // check main division equality (q << shift) + r == x + CHECK((qv << shift) + rv == xv); + MRInt rval(rv); + // check remainder range + switch (round_mode) { + case 1: + rval.negate(); // fallthrough + case -1: + CHECK(mr_ufits_bits(rval, shift)); + break; + case 0: + CHECK(mr_fits_bits(rval, shift)); + } + } +} + +void check_shift_ops() { + std::cerr << "check left/right shift ops" << std::endl; + for (int idx = min_spec_int; idx <= max_spec_int; idx++) { + //for (int idx : {-52240, -52239, -52238, -3, -2, -1, 0, 1, 2, 3, 52238, 52239, 52240}) { + const auto& xv = SpecIntB[idx - min_spec_int]; + MRInt mval(xv); + if (!(idx % 1000)) { + std::cerr << "# " << idx << " : " << mval << std::endl; + } + for (int i = 0; i <= 256; i++) { + check_shift_ops_on(i, SpecInt[idx - min_spec_int], xv, mval); + } + } +} + +void check_remainder_range(BInt& rv, const BInt& dv, int rmode = -1) { + if (rmode > 0) { + rv.negate(); + } else if (!rmode) { + rv *= 2; + } + MRInt d(dv), r(rv); + int ds = d.sgn(), rs = r.sgn(); + //std::cerr << "rmode=" << rmode << " ds=" << ds << " rs=" << rs << " d=" << d << " r=" << r << std::endl; + if (!rs) { + return; + } + if (rmode) { + // must have 0 < r < d or 0 > r > d + //if (rs != ds) std::cerr << "iter=" << cur_iteration << " : rmode=" << rmode << " ds=" << ds << " rs=" << rs << " d=" << d << " r=" << r << std::endl; + CHECK(rs == ds); + CHECK(ds * r.cmp(d) < 0); + } else { + // must have -d <= r < d or -d >= r > d + if (rs == -ds) { + r.negate(); + CHECK(ds * r.cmp(d) <= 0); + } else { + CHECK(ds * r.cmp(d) < 0); + } + } +} + +void check_divmod(td::RefInt256 x, const BInt& xv, long long xl, td::RefInt256 y, const BInt& yv, long long yl, + int rmode = -2) { + if (rmode < -1) { + //IFDEBUG std::cerr << " divide " << x << " / " << y << std::endl; + for (rmode = -1; rmode <= 1; rmode++) { + check_divmod(x, xv, xl, y, yv, yl, rmode); + } + return; + } + auto dm = td::divmod(x, y, rmode); + auto q = std::move(dm.first), r = std::move(dm.second); + if (!yl) { + // division by zero + CHECK(q.not_null() && !q->is_valid() && r.not_null() && !r->is_valid()); + return; + } + CHECK(q.not_null() && q->is_valid() && r.not_null() && r->is_valid()); + CHECK_EQ(x, y * q + r); + BInt qv = extract_value(*q), rv = extract_value(*r); + CHECK(xv == yv * qv + rv); + //IFDEBUG std::cerr << " quot=" << q << " rem=" << r << std::endl; + check_remainder_range(rv, yv, rmode); + if (yl != ll_min && rmode == -1) { + // check divmod_short() + auto qq = x; + auto rem = qq.write().divmod_short(yl); + qq.write().normalize(); + CHECK(qq->is_valid()); + CHECK_EQ(qq, q); + CHECK(r == rem); + if (xl != ll_min) { + auto dm = std::lldiv(xl, yl); + if (dm.rem && (dm.rem ^ yl) < 0) { + dm.rem += yl; + dm.quot--; + } + CHECK(q == dm.quot); + CHECK(r == dm.rem); + } + } +} + +void check_binary_ops_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv) { + bool x_small = x->fits_bits(62), y_small = y->fits_bits(62); // not 63 + long long xl = x_small ? x->to_long() : ll_min, yl = y_small ? y->to_long() : ll_min; + if (x_small) { + CHECK(x == xl); + } + if (y_small) { + CHECK(y == yl); + } + // ADD, ADDR + auto z = x + y, w = y + x; + CHECK_EQ(z, w); + check_res(z, xv + yv); + // ADDCONST + if (y_small) { + CHECK_EQ(z, x + yl); + } + if (x_small) { + CHECK_EQ(z, y + xl); + } + if (x_small && y_small) { + CHECK_EQ(z, xl + yl); + } + // SUB + z = x - y; + check_res(z, xv - yv); + // SUBCONST + if (y_small) { + CHECK_EQ(z, x - yl); + if (x_small) { + CHECK_EQ(z, xl - yl); + } + } + // SUBR + z = y - x; + check_res(z, yv - xv); + if (x_small) { + CHECK_EQ(z, y - xl); + if (y_small) { + CHECK_EQ(z, yl - xl); + } + } + // CMP + MRInt xmr(xv), ymr(yv); + int cmpv = xmr.cmp(ymr); + CHECK(td::cmp(x, y) == cmpv); + CHECK(td::cmp(y, x) == -cmpv); + if (y_small) { + CHECK(td::cmp(x, yl) == cmpv); + } + if (x_small) { + CHECK(td::cmp(y, xl) == -cmpv); + } + if (x_small && y_small) { + CHECK(cmpv == (xl < yl ? -1 : (xl > yl ? 1 : 0))); + } + // MUL + z = x * y; + BInt zv = xv * yv; + check_res(z, zv); + CHECK_EQ(z, y * x); + // MULCONST + if (y_small) { + CHECK_EQ_NAN(z, x * yl); + } + if (x_small) { + CHECK_EQ_NAN(z, y * xl); + } + if (x_small && y_small && (!yl || std::abs(xl) <= ll_max / std::abs(yl))) { + CHECK_EQ(z, xl * yl); + } + // DIVMOD + if (z->fits_bits(257)) { + int adj = 2 * rand_int(-2, 2) - (int)z->is_odd(); + z += adj; + z >>= 1; + zv += adj; + zv >>= 1; + // z is approximately x * y / 2; divide by y + check_divmod(z, zv, z->fits_bits(62) ? z->to_long() : ll_min, y, yv, yl); + } + check_divmod(x, xv, xl, y, yv, yl); +} + +void finish_check_muldivmod(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, td::RefInt256 z, + const BInt& zv, td::RefInt256 q, td::RefInt256 r, int rmode) { + static constexpr double eps = 1e-14; + CHECK(q.not_null() && r.not_null()); + //std::cerr << " muldivmod: " << xv << " * " << yv << " / " << zv << " (round " << rmode << ") = " << q << " " << r << std::endl; + if (!zv) { + // division by zero + CHECK(!q->is_valid() && !r->is_valid()); + return; + } + CHECK(r->is_valid()); // remainder always exists if y != 0 + BInt xyv = xv * yv, rv = extract_value(*r); + MRInt xy_mr(xyv), z_mr(zv); + double q0 = (double)xy_mr / (double)z_mr; + if (std::abs(q0) < 1.01 * dbl_pow256) { + // result more or less in range + CHECK(q->is_valid()); + } else if (!q->is_valid()) { + // result out of range, NaN is an acceptable answer + // check that x * y - r is divisible by z + xyv -= rv; + xyv /= zv; + xy_mr = xyv; + double q1 = (double)xy_mr; + CHECK(std::abs(q1 - q0) < eps * std::abs(q0)); + } else { + BInt qv = extract_value(*q); + // must have x * y = z * q + r + CHECK(xv * yv == zv * qv + rv); + } + // check that r is in correct range [0, z) or [0, -z) or [-z/2, z/2) + check_remainder_range(rv, zv, rmode); +} + +void check_muldivmod_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, td::RefInt256 z, + const BInt& zv, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_muldivmod_on(x, xv, y, yv, z, zv, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // MULDIVMOD + auto qr = td::muldivmod(x, y, z, rmode); + finish_check_muldivmod(std::move(x), xv, std::move(y), yv, std::move(z), zv, std::move(qr.first), + std::move(qr.second), rmode); +} + +void check_mul_rshift_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, int shift, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_mul_rshift_on(x, xv, y, yv, shift, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // MULRSHIFTMOD + typename td::BigInt256::DoubleInt tmp{0}; + tmp.add_mul(*x, *y); + typename td::BigInt256::DoubleInt tmp2{tmp}; + tmp2.rshift(shift, rmode).normalize(); + tmp.normalize().mod_pow2(shift, rmode).normalize(); + finish_check_muldivmod(std::move(x), xv, std::move(y), yv, {}, BInt::pow2(shift), td::make_refint(tmp2), + td::make_refint(tmp), rmode); +} + +void check_lshift_div_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, int shift, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_lshift_div_on(x, xv, y, yv, shift, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // LSHIFTDIV + typename td::BigInt256::DoubleInt tmp{*x}, quot; + tmp <<= shift; + tmp.mod_div(*y, quot, rmode); + quot.normalize(); + finish_check_muldivmod(std::move(x), xv, {}, BInt::pow2(shift), std::move(y), yv, td::make_refint(quot), + td::make_refint(tmp), rmode); +} + +void check_random_ops() { + constexpr long long chk_it = 100000; + std::cerr << "check random ops (" << iterations << " iterations)" << std::endl; + BInt xv, yv, zv; + unsigned char xbin[64], ybin[64], zbin[64]; + for (cur_iteration = 0; cur_iteration < iterations; cur_iteration++) { + auto x = make_random_int0(xv, xbin); + if (!(cur_iteration % 10000)) { + std::cerr << "#" << cur_iteration << ": check on " << xv << " = " << ShowBin(xbin) << " = " << x->as_any_int() + << std::endl; + } + check_one_int_repr(x, cur_iteration < chk_it ? -1 : 0, -1, &xv, xbin); + MRInt xmr(xv); + if (!x->fits_bits(257)) { + continue; + } + check_unary_ops_on(x, xv); + for (int j = 0; j < 10; j++) { + int shift = rand_int(0, 256); + //std::cerr << "check shift by " << shift << std::endl; + check_shift_ops_on(shift, x, xv, xmr); + auto y = make_random_int(yv, ybin); + //std::cerr << " y = " << y << " = " << yv << " = " << ShowBin(ybin) << " = " << y->as_any_int() << std::endl; + check_one_int_repr(y, 0, 1, &yv, ybin); + check_binary_ops_on(x, xv, y, yv); + //std::cerr << " *>> " << shift << std::endl; + check_mul_rshift_on(x, xv, y, yv, shift); + //std::cerr << " <as_any_int() << std::endl; + check_muldivmod_on(x, xv, y, yv, z, zv); + } + } +} + +void check_special() { + std::cerr << "run special tests" << std::endl; + check_divmod((td::make_refint(-1) << 207) - 1, BInt::negpow2(207) - 1, ll_min, (td::make_refint(1) << 207) - 1, + BInt::pow2(207) - 1, ll_min); +} + +int main(int argc, char* const argv[]) { + bool do_check_shift_ops = false; + int i; + while ((i = getopt(argc, argv, "hSs:i:")) != -1) { + switch (i) { + case 'S': + do_check_shift_ops = true; + break; + case 's': + Random.seed(atoll(optarg)); + break; + case 'i': + iterations = atoll(optarg); + break; + default: + std::cerr << "unknown option: " << (char)i << std::endl; + // fall through + case 'h': + std::cerr << "usage:\t" << argv[0] << " [-S] [-i] [-s]" << std::endl; + return 2; + } + } + modint::init(); + init_aux(); + init_check_special_ints(); + check_pow2_ops(); + check_unary_ops(); + if (do_check_shift_ops) { + check_shift_ops(); + } + check_special(); + check_random_ops(); + return 0; +} diff --git a/crypto/vm/arithops.cpp b/crypto/vm/arithops.cpp index 823b4408..2831944b 100644 --- a/crypto/vm/arithops.cpp +++ b/crypto/vm/arithops.cpp @@ -387,18 +387,16 @@ int exec_muldivmod(VmState* st, unsigned args, int quiet) { auto z = stack.pop_int(); auto y = stack.pop_int(); auto x = stack.pop_int(); - typename td::BigInt256::DoubleInt tmp{0}; + typename td::BigInt256::DoubleInt tmp{0}, quot; tmp.add_mul(*x, *y); auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); + tmp.mod_div(*z, quot, round_mode); switch ((args >> 2) & 3) { case 1: - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), quiet); + stack.push_int_quiet(td::make_refint(quot.normalize()), quiet); break; case 3: - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), quiet); + stack.push_int_quiet(td::make_refint(quot.normalize()), quiet); // fallthrough case 2: stack.push_int_quiet(td::make_refint(tmp), quiet); @@ -459,7 +457,7 @@ int exec_mulshrmod(VmState* st, unsigned args, int mode) { } // fallthrough case 2: - tmp.mod_pow2(z, round_mode).normalize(); + tmp.normalize().mod_pow2(z, round_mode).normalize(); stack.push_int_quiet(td::make_refint(tmp), mode & 1); break; } @@ -520,21 +518,17 @@ int exec_shldivmod(VmState* st, unsigned args, int mode) { } auto z = stack.pop_int(); auto x = stack.pop_int(); - typename td::BigInt256::DoubleInt tmp{*x}; + typename td::BigInt256::DoubleInt tmp{*x}, quot; tmp <<= y; switch ((args >> 2) & 3) { case 1: { - auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), mode & 1); + tmp.mod_div(*z, quot, round_mode); + stack.push_int_quiet(td::make_refint(quot.normalize()), mode & 1); break; } case 3: { - auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), mode & 1); + tmp.mod_div(*z, quot, round_mode); + stack.push_int_quiet(td::make_refint(quot.normalize()), mode & 1); stack.push_int_quiet(td::make_refint(tmp), mode & 1); break; } diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index c53595ff..8ff244d0 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -475,15 +475,128 @@ std::string BagOfCells::extract_string() const { return std::string{serialized.data(), serialized.data() + serialized.size()}; } -void BagOfCells::store_uint(unsigned long long value, unsigned bytes) { - unsigned char* ptr = store_ptr += bytes; - store_chk(); - while (bytes) { - *--ptr = value & 0xff; - value >>= 8; - --bytes; +namespace { +struct BufferWriter { + BufferWriter(unsigned char* store_start, unsigned char* store_end) + : store_start(store_start), store_ptr(store_start), store_end(store_end) {} + + size_t position() const { + return store_ptr - store_start; } - DCHECK(!bytes); + size_t remaining() const { + return store_end - store_ptr; + } + void chk() const { + DCHECK(store_ptr <= store_end); + } + bool empty() const { + return store_ptr == store_end; + } + void store_uint(unsigned long long value, unsigned bytes) { + unsigned char* ptr = store_ptr += bytes; + chk(); + while (bytes) { + *--ptr = value & 0xff; + value >>= 8; + --bytes; + } + DCHECK(!bytes); + } + void store_bytes(unsigned char const* data, size_t s) { + store_ptr += s; + chk(); + memcpy(store_ptr - s, data, s); + } + unsigned get_crc32() const { + return td::crc32c(td::Slice{store_start, store_ptr}); + } + + private: + unsigned char* store_start; + unsigned char* store_ptr; + unsigned char* store_end; +}; + +struct FileWriter { + FileWriter(td::FileFd& fd, size_t expected_size) + : fd(fd), expected_size(expected_size) {} + + ~FileWriter() { + flush(); + } + + size_t position() const { + return flushed_size + writer.position(); + } + size_t remaining() const { + return expected_size - position(); + } + void chk() const { + DCHECK(position() <= expected_size); + } + bool empty() const { + return remaining() == 0; + } + void store_uint(unsigned long long value, unsigned bytes) { + flush_if_needed(bytes); + writer.store_uint(value, bytes); + } + void store_bytes(unsigned char const* data, size_t s) { + flush_if_needed(s); + writer.store_bytes(data, s); + } + unsigned get_crc32() const { + unsigned char const* start = buf.data(); + unsigned char const* end = start + writer.position(); + return td::crc32c_extend(current_crc32, td::Slice(start, end)); + } + + td::Status finalize() { + flush(); + return std::move(res); + } + + private: + void flush_if_needed(size_t s) { + DCHECK(s <= BUF_SIZE); + if (s > BUF_SIZE - writer.position()) { + flush(); + } + } + + void flush() { + chk(); + unsigned char* start = buf.data(); + unsigned char* end = start + writer.position(); + if (start == end) { + return; + } + flushed_size += end - start; + current_crc32 = td::crc32c_extend(current_crc32, td::Slice(start, end)); + if (res.is_ok()) { + while (end > start) { + auto R = fd.write(td::Slice(start, end)); + if (R.is_error()) { + res = R.move_as_error(); + break; + } + size_t s = R.move_as_ok(); + start += s; + } + } + writer = BufferWriter(buf.data(), buf.data() + buf.size()); + } + + td::FileFd& fd; + size_t expected_size; + size_t flushed_size = 0; + unsigned current_crc32 = td::crc32c(td::Slice()); + + static const size_t BUF_SIZE = 1 << 22; + std::vector buf = std::vector(BUF_SIZE, '\0'); + BufferWriter writer = BufferWriter(buf.data(), buf.data() + buf.size()); + td::Status res = td::Status::OK(); +}; } //serialized_boc#672fb0ac has_idx:(## 1) has_crc32c:(## 1) @@ -497,13 +610,16 @@ void BagOfCells::store_uint(unsigned long long value, unsigned bytes) { // index:(cells * ##(off_bytes * 8)) // cell_data:(tot_cells_size * [ uint8 ]) // = BagOfCells; -std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) { - std::size_t size_est = estimate_serialized_size(mode); - if (!size_est || size_est > buff_size) { - return 0; - } - init_store(buffer, buffer + size_est); - store_uint(info.magic, 4); +template +std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { + auto store_ref = [&](unsigned long long value) { + writer.store_uint(value, info.ref_byte_size); + }; + auto store_offset = [&](unsigned long long value) { + writer.store_uint(value, info.offset_byte_size); + }; + + writer.store_uint(info.magic, 4); td::uint8 byte{0}; if (info.has_index) { @@ -520,9 +636,9 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz return 0; } byte |= static_cast(info.ref_byte_size); - store_uint(byte, 1); + writer.store_uint(byte, 1); - store_uint(info.offset_byte_size, 1); + writer.store_uint(info.offset_byte_size, 1); store_ref(cell_count); store_ref(root_count); store_ref(0); @@ -532,7 +648,7 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz DCHECK(k >= 0 && k < cell_count); store_ref(k); } - DCHECK(store_ptr - buffer == (long long)info.index_offset); + DCHECK(writer.position() == info.index_offset); DCHECK((unsigned)cell_count == cell_list_.size()); if (info.has_index) { std::size_t offs = 0; @@ -551,8 +667,8 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz } DCHECK(offs == info.data_size); } - DCHECK(store_ptr - buffer == (long long)info.data_offset); - unsigned char* keep_ptr = store_ptr; + DCHECK(writer.position() == info.data_offset); + size_t keep_position = writer.position(); for (int i = 0; i < cell_count; ++i) { const auto& dc_info = cell_list_[cell_count - 1 - i]; const Ref& dc = dc_info.dc_ref; @@ -560,9 +676,9 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { with_hash = true; } - int s = dc->serialize(store_ptr, 256, with_hash); - store_ptr += s; - store_chk(); + unsigned char buf[256]; + int s = dc->serialize(buf, 256, with_hash); + writer.store_bytes(buf, s); DCHECK(dc->size_refs() == dc_info.ref_num); // std::cerr << (dc_info.is_special() ? '*' : ' ') << i << '<' << (int)dc_info.wt << ">:"; for (unsigned j = 0; j < dc_info.ref_num; ++j) { @@ -573,16 +689,38 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz } // std::cerr << std::endl; } - store_chk(); - DCHECK(store_ptr - keep_ptr == (long long)info.data_size); - DCHECK(store_end - store_ptr == (info.has_crc32c ? 4 : 0)); + writer.chk(); + DCHECK(writer.position() - keep_position == info.data_size); + DCHECK(writer.remaining() == (info.has_crc32c ? 4 : 0)); if (info.has_crc32c) { - // compute crc32c of buffer .. store_ptr - unsigned crc = td::crc32c(td::Slice{buffer, store_ptr}); - store_uint(td::bswap32(crc), 4); + unsigned crc = writer.get_crc32(); + writer.store_uint(td::bswap32(crc), 4); } - DCHECK(store_empty()); - return store_ptr - buffer; + DCHECK(writer.empty()); + return writer.position(); +} + +std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) { + std::size_t size_est = estimate_serialized_size(mode); + if (!size_est || size_est > buff_size) { + return 0; + } + BufferWriter writer{buffer, buffer + size_est}; + return serialize_to_impl(writer, mode); +} + +td::Status BagOfCells::serialize_to_file(td::FileFd& fd, int mode) { + std::size_t size_est = estimate_serialized_size(mode); + if (!size_est) { + return td::Status::Error("no cells to serialize to this bag of cells"); + } + FileWriter writer{fd, size_est}; + size_t s = serialize_to_impl(writer, mode); + TRY_STATUS(writer.finalize()); + if (s != size_est) { + return td::Status::Error("error while serializing a bag of cells: actual serialized size differs from estimated"); + } + return td::Status::OK(); } unsigned long long BagOfCells::Info::read_int(const unsigned char* ptr, unsigned bytes) { diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index cc277a8c..02078e27 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -23,6 +23,7 @@ #include "td/utils/buffer.h" #include "td/utils/HashMap.h" #include "td/utils/HashSet.h" +#include "td/utils/port/FileFd.h" namespace vm { using td::Ref; @@ -216,8 +217,6 @@ class BagOfCells { int max_depth{1024}; Info info; unsigned long long data_bytes{0}; - unsigned char* store_ptr{nullptr}; - unsigned char* store_end{nullptr}; td::HashMap cells; struct CellInfo { Ref dc_ref; @@ -267,6 +266,9 @@ class BagOfCells { std::string serialize_to_string(int mode = 0); td::Result serialize_to_slice(int mode = 0); std::size_t serialize_to(unsigned char* buffer, std::size_t buff_size, int mode = 0); + td::Status serialize_to_file(td::FileFd& fd, int mode = 0); + template + std::size_t serialize_to_impl(WriterT& writer, int mode = 0); std::string extract_string() const; td::Result deserialize(const td::Slice& data, int max_roots = default_max_roots); @@ -295,23 +297,6 @@ class BagOfCells { cell_list_.clear(); } td::uint64 compute_sizes(int mode, int& r_size, int& o_size); - void init_store(unsigned char* from, unsigned char* to) { - store_ptr = from; - store_end = to; - } - void store_chk() const { - DCHECK(store_ptr <= store_end); - } - bool store_empty() const { - return store_ptr == store_end; - } - void store_uint(unsigned long long value, unsigned bytes); - void store_ref(unsigned long long value) { - store_uint(value, info.ref_byte_size); - } - void store_offset(unsigned long long value) { - store_uint(value, info.offset_byte_size); - } void reorder_cells(); int revisit(int cell_idx, int force = 0); unsigned long long get_idx_entry_raw(int index); diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 038640a5..fb2f4d1e 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -604,6 +604,7 @@ Ref VmState::load_library(td::ConstBitPtr hash) { return lib; } } + missing_library = hash; return {}; } diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index 0fef9642..44aa7f23 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -96,6 +96,7 @@ class VmState final : public VmStateInterface { td::int64 loaded_cells_count{0}; int stack_trace{0}, debug_off{0}; bool chksig_always_succeed{false}; + td::ConstBitPtr missing_library{0}; public: enum { @@ -321,6 +322,9 @@ class VmState final : public VmStateInterface { Ref ref_to_cont(Ref cell) const { return td::make_ref(load_cell_slice_ref(std::move(cell)), get_cp()); } + td::ConstBitPtr get_missing_library() const { + return missing_library; + } private: void init_cregs(bool same_c3 = false, bool push_0 = true); diff --git a/doc/tvm.tex b/doc/tvm.tex index 4f887ac2..71aa7abc 100644 --- a/doc/tvm.tex +++ b/doc/tvm.tex @@ -1531,6 +1531,7 @@ Examples: \item {\tt A934$tt$} --- same as {\tt RSHIFT $tt+1$}: ($x$ -- $\lfloor x\cdot 2^{-tt-1}\rfloor$). \item {\tt A938$tt$} --- {\tt MODPOW2 $tt+1$}: ($x$ -- $x\bmod 2^{tt+1}$). \item {\tt A985} --- {\tt MULDIVR} ($x$ $y$ $z$ -- $q'$), where $q'=\lfloor xy/z+1/2\rfloor$. +\item {\tt A988} --- {\tt MULMOD} ($x$ $y$ $z$ -- $r$), where $r=xy\bmod z=xy-qz$, $q=\lfloor xy/z\rfloor$. This operation always succeeds for $z\neq0$ and returns the correct value of~$r$, even if the intermediate result $xy$ or the quotient $q$ do not fit into 257 bits. \item {\tt A98C} --- {\tt MULDIVMOD} ($x$ $y$ $z$ -- $q$ $r$), where $q:=\lfloor x\cdot y/z\rfloor$, $r:=x\cdot y\bmod z$ (same as {\tt */MOD} in Forth). \item {\tt A9A4} --- {\tt MULRSHIFT} ($x$ $y$ $z$ -- $\lfloor xy\cdot2^{-z}\rfloor$) for $0\leq z\leq 256$. \item {\tt A9A5} --- {\tt MULRSHIFTR} ($x$ $y$ $z$ -- $\lfloor xy\cdot2^{-z}+1/2\rfloor$) for $0\leq z\leq 256$. @@ -1576,10 +1577,12 @@ We opted to make all arithmetic operations ``non-quiet'' (signaling) by default, \begin{itemize} \item {\tt B7xx} --- {\tt QUIET} prefix, transforming any arithmetic operation into its ``quiet'' variant, indicated by prefixing a {\tt Q} to its mnemonic. Such operations return {\tt NaN}s instead of throwing integer overflow exceptions if the results do not fit in {\it Integer\/}s, or if one of their arguments is a {\tt NaN}. Notice that this does not extend to shift amounts and other parameters that must be within a small range (e.g., 0--1023). Also notice that this does not disable type-checking exceptions if a value of a type other than {\it Integer\/} is supplied. \item {\tt B7A0} --- {\tt QADD} ($x$ $y$ -- $x+y$), always works if $x$ and $y$ are {\it Integer\/}s, but returns a {\tt NaN} if the addition cannot be performed. +\item {\tt B7A8} --- {\tt QMUL} ($x$ $y$ -- $xy$), returns the product of $x$ and $y$ if $-2^{256}\leq xy<2^{256}$. Otherwise returns a {\tt NaN}, even if $x=0$ and $y$ is a {\tt NaN}. \item {\tt B7A904} --- {\tt QDIV} ($x$ $y$ -- $\lfloor x/y\rfloor$), returns a {\tt NaN} if $y=0$, or if $y=-1$ and $x=-2^{256}$, or if either of $x$ or $y$ is a {\tt NaN}. +\item {\tt B7A98C} --- {\tt QMULDIVMOD} ($x$ $y$ $z$ -- $q$ $r$), where $q:=\lfloor x\cdot y/z\rfloor$, $r:=x\cdot y\bmod z$. If $z=0$, or if at least one of $x$, $y$, or $z$ is a {\tt NaN}, both $q$ and $r$ are set to {\tt NaN}. Otherwise the correct value of $r$ is always returned, but $q$ is replaced with {\tt NaN} if $q<-2^{256}$ or $q\geq2^{256}$. \item {\tt B7B0} --- {\tt QAND} ($x$ $y$ -- $x\&y$), bitwise ``and'' (similar to {\tt AND}), but returns a {\tt NaN} if either $x$ or $y$ is a {\tt NaN} instead of throwing an integer overflow exception. However, if one of the arguments is zero, and the other is a {\tt NaN}, the result is zero. \item {\tt B7B1} --- {\tt QOR} ($x$ $y$ -- $x\vee y$), bitwise ``or''. If $x=-1$ or $y=-1$, the result is always $-1$, even if the other argument is a {\tt NaN}. -\item {\tt B7B507} --- {\tt QUFITS 8} ($x$ -- $x'$), checks whether $x$ is an unsigned byte (i.e., whether $0\leq x<2^8$), and replaces $x$ with a {\tt NaN} if this is not the case; leaves $x$ intact otherwise (i.e., if $x$ is an unsigned byte). +\item {\tt B7B507} --- {\tt QUFITS 8} ($x$ -- $x'$), checks whether $x$ is an unsigned byte (i.e., whether $0\leq x<2^8$), and replaces $x$ with a {\tt NaN} if this is not the case; leaves $x$ intact otherwise (i.e., if $x$ is an unsigned byte or a {\tt NaN}). \end{itemize} \mysubsection{Comparison primitives} diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 1ccfec86..f472af21 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -4261,6 +4261,8 @@ int main(int argc, char* argv[]) { }); p.add_option('p', "pub", "remote public key", [&](td::Slice arg) { td::actor::send_closure(x, &TestNode::set_public_key, td::BufferSlice{arg}); }); + p.add_option('b', "b64", "remote public key as base64", + [&](td::Slice arg) { td::actor::send_closure(x, &TestNode::decode_public_key, td::BufferSlice{arg}); }); p.add_option('d', "daemonize", "set SIGHUP", [&]() { td::set_signal_handler(td::SignalType::HangUp, [](int sig) { #if TD_DARWIN || TD_LINUX diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 7184652a..00d8bd07 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -394,6 +394,18 @@ class TestNode : public td::actor::Actor { } remote_public_key_ = R.move_as_ok(); } + void decode_public_key(td::BufferSlice b64_key) { + auto R = [&]() -> td::Result { + std::string key_bytes = {(char)0xc6, (char)0xb4, (char)0x13, (char)0x48}; + key_bytes = key_bytes + td::base64_decode(b64_key.as_slice().str()).move_as_ok(); + return ton::PublicKey::import(key_bytes); + }(); + + if (R.is_error()) { + LOG(FATAL) << "bad b64 server public key: " << R.move_as_error(); + } + remote_public_key_ = R.move_as_ok(); + } void set_fail_timeout(td::Timestamp ts) { fail_timeout_ = ts; alarm_timestamp().relax(fail_timeout_); diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index bade50e7..1c05f705 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -17,8 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "overlay-broadcast.hpp" +#include "adnl/adnl-node-id.hpp" +#include "common/util.h" #include "overlay.hpp" #include "keys/encryptor.h" +#include "td/actor/PromiseFuture.h" +#include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/port/Stat.h" namespace ton { @@ -33,7 +39,13 @@ td::Status BroadcastSimple::check_duplicate() { } td::Status BroadcastSimple::check_source() { - return overlay_->check_source_eligible(source_, cert_.get(), data_size()); + auto r = overlay_->check_source_eligible(source_, cert_.get(), data_size(), false); + if (r == BroadcastCheckResult::Forbidden) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + + is_valid_ = r == BroadcastCheckResult::Allowed; + return td::Status::OK(); } td::BufferSlice BroadcastSimple::to_sign() { @@ -66,6 +78,14 @@ td::Status BroadcastSimple::distribute() { return td::Status::OK(); } +void BroadcastSimple::broadcast_checked(td::Result R) { + if (R.is_error()) { + return; + } + is_valid_ = true; + run_continue().ignore(); +} + tl_object_ptr BroadcastSimple::tl() const { return create_tl_object(source_.tl(), cert_ ? cert_->tl() : Certificate::empty_tl(), flags_, data_.clone(), date_, signature_.clone()); @@ -75,6 +95,25 @@ td::BufferSlice BroadcastSimple::serialize() { return serialize_tl_object(tl(), true); } +td::Status BroadcastSimple::run_continue() { + TRY_STATUS(distribute()); + deliver(); + return td::Status::OK(); +} + +td::Status BroadcastSimple::run() { + TRY_STATUS(run_checks()); + if (!is_valid_) { + auto P = td::PromiseCreator::lambda( + [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result R) mutable { + td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(R)); + }); + overlay_->check_broadcast(source_.compute_short_id(), data_.clone(), std::move(P)); + return td::Status::OK(); + } + return run_continue(); +} + td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr broadcast) { auto src = PublicKey{broadcast->src_}; auto data_hash = sha256_bits256(broadcast->data_.as_slice()); @@ -86,7 +125,7 @@ td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr(broadcast_hash, src, std::move(cert), broadcast->flags_, std::move(broadcast->data_), broadcast->date_, - std::move(broadcast->signature_), overlay); + std::move(broadcast->signature_), false, overlay); TRY_STATUS(B->run()); overlay->register_simple_broadcast(std::move(B)); return td::Status::OK(); @@ -100,7 +139,7 @@ td::Status BroadcastSimple::create_new(td::actor::ActorId overlay, auto date = static_cast(td::Clocks::system()); auto B = std::make_unique(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date, - td::BufferSlice{}, nullptr); + td::BufferSlice{}, false, nullptr); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index 665c269d..da29695a 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -18,9 +18,16 @@ */ #pragma once +#include "adnl/adnl-local-id.h" +#include "adnl/adnl-node-id.hpp" #include "auto/tl/ton_api.h" +#include "common/refcnt.hpp" #include "overlay/overlay.h" +#include "td/actor/PromiseFuture.h" #include "td/utils/List.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" namespace ton { @@ -38,6 +45,7 @@ class BroadcastSimple : public td::ListNode { td::BufferSlice data_; td::uint32 date_; td::BufferSlice signature_; + bool is_valid_{false}; OverlayImpl *overlay_; @@ -52,7 +60,7 @@ class BroadcastSimple : public td::ListNode { public: BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr cert, - td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, + td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid, OverlayImpl *overlay) : broadcast_hash_(broadcast_hash) , source_(std::move(source)) @@ -61,6 +69,7 @@ class BroadcastSimple : public td::ListNode { , data_(std::move(data)) , date_(date) , signature_(std::move(signature)) + , is_valid_(is_valid) , overlay_(overlay) { } @@ -80,17 +89,14 @@ class BroadcastSimple : public td::ListNode { } void deliver(); - td::Status run() { - TRY_STATUS(run_checks()); - TRY_STATUS(distribute()); - deliver(); - return td::Status::OK(); - } + td::Status run(); + td::Status run_continue(); tl_object_ptr tl() const; td::BufferSlice serialize(); void update_overlay(OverlayImpl *overlay); + void broadcast_checked(td::Result R); static td::Status create(OverlayImpl *overlay, tl_object_ptr broadcast); static td::Status create_new(td::actor::ActorId overlay, td::actor::ActorId keyring, diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index e368c620..0ff53346 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -54,10 +54,20 @@ td::Status OverlayFecBroadcastPart::check_duplicate() { } td::Status OverlayFecBroadcastPart::check_source() { - TRY_STATUS(overlay_->check_source_eligible(source_, cert_.get(), broadcast_size_)); + auto r = overlay_->check_source_eligible(source_, cert_.get(), broadcast_size_, true); + if (r == BroadcastCheckResult::Forbidden) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + + if (r == BroadcastCheckResult::NeedCheck) { + untrusted_ = true; + return td::Status::OK(); + } + if (bcast_) { TRY_STATUS(bcast_->is_eligible_sender(source_)); } + return td::Status::OK(); } @@ -68,6 +78,7 @@ td::Status OverlayFecBroadcastPart::check_signature() { } td::Status OverlayFecBroadcastPart::run_checks() { + TRY_STATUS(check_time()); TRY_STATUS(check_duplicate()); TRY_STATUS(check_source()); @@ -75,7 +86,52 @@ td::Status OverlayFecBroadcastPart::run_checks() { return td::Status::OK(); } +void BroadcastFec::broadcast_checked(td::Result R) { + if (R.is_error()) { + return; + } + overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone()); + auto manager = overlay_->overlay_manager(); + while (!parts_.empty()) { + distribute_part(parts_.begin()->first); + } +} + +// Do we need status here?? +td::Status BroadcastFec::distribute_part(td::uint32 seqno) { + auto i = parts_.find(seqno); + if (i == parts_.end()) { + // should not get here + return td::Status::OK(); + } + auto tls = std::move(i->second); + parts_.erase(i); + td::BufferSlice data_short = std::move(tls.first); + td::BufferSlice data = std::move(tls.second); + + auto nodes = overlay_->get_neighbours(5); + auto manager = overlay_->overlay_manager(); + + for (auto &n : nodes) { + if (neighbour_completed(n)) { + continue; + } + if (neighbour_received(n)) { + td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(), + data_short.clone()); + } else { + if (hash_.count_leading_zeroes() >= 12) { + VLOG(OVERLAY_INFO) << "broadcast " << hash_ << ": sending part " << seqno << " to " << n; + } + td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(), + data.clone()); + } + } + return td::Status::OK(); +} + td::Status OverlayFecBroadcastPart::apply() { + if (!bcast_) { bcast_ = overlay_->get_fec_broadcast(broadcast_hash_); } @@ -98,7 +154,8 @@ td::Status OverlayFecBroadcastPart::apply() { } if (!bcast_->finalized()) { - TRY_STATUS(bcast_->add_part(seqno_, data_.clone())); + bcast_->set_overlay(overlay_); + TRY_STATUS(bcast_->add_part(seqno_, data_.clone(), export_serialized_short(), export_serialized())); auto R = bcast_->finish(); if (R.is_error()) { auto S = R.move_as_error(); @@ -106,44 +163,22 @@ td::Status OverlayFecBroadcastPart::apply() { return S; } } else { - overlay_->deliver_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok()); + if(untrusted_) { + auto P = td::PromiseCreator::lambda( + [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result RR) mutable { + td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(RR)); + }); + overlay_->check_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok(), std::move(P)); + } else { + overlay_->deliver_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok()); + } } } - return td::Status::OK(); } td::Status OverlayFecBroadcastPart::distribute() { - auto B = export_serialized(); - auto nodes = overlay_->get_neighbours(5); - - auto manager = overlay_->overlay_manager(); - - td::BufferSlice data; - td::BufferSlice data_short; - - for (auto &n : nodes) { - if (bcast_->neighbour_completed(n)) { - continue; - } - if (bcast_->neighbour_received(n)) { - if (data_short.size() == 0) { - data_short = export_serialized_short(); - } - td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(), - data_short.clone()); - } else { - if (data.size() == 0) { - data = export_serialized(); - } - - if (broadcast_hash_.count_leading_zeroes() >= 12) { - VLOG(OVERLAY_INFO) << "broadcast " << broadcast_hash_ << ": sending part " << part_hash_ << " to " << n; - } - td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(), - data.clone()); - } - } + TRY_STATUS(bcast_->distribute_part(seqno_)); return td::Status::OK(); } @@ -179,7 +214,6 @@ td::BufferSlice OverlayFecBroadcastPart::to_sign() { td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, tl_object_ptr broadcast) { TRY_STATUS(overlay->check_date(broadcast->date_)); - auto source = PublicKey{broadcast->src_}; auto part_data_hash = sha256_bits256(broadcast->data_.as_slice()); diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 6d30d47c..f27e69fb 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -82,13 +82,17 @@ class BroadcastFec : public td::ListNode { } } - td::Status add_part(td::uint32 seqno, td::BufferSlice data) { + td::Status add_part(td::uint32 seqno, td::BufferSlice data, + td::BufferSlice serialized_fec_part_short, + td::BufferSlice serialized_fec_part) { CHECK(decoder_); td::fec::Symbol s; s.id = seqno; s.data = std::move(data); decoder_->add_symbol(std::move(s)); + parts_[seqno] = std::pair(std::move(serialized_fec_part_short), + std::move(serialized_fec_part)); return td::Status::OK(); } @@ -106,6 +110,7 @@ class BroadcastFec : public td::ListNode { CHECK(encoder_ != nullptr); ready_ = true; decoder_ = nullptr; + data_ = D.data.clone(); return std::move(D.data); } @@ -185,6 +190,13 @@ class BroadcastFec : public td::ListNode { } } + void broadcast_checked(td::Result R); + void set_overlay(OverlayImpl *overlay) { + overlay_ = overlay; + } + + td::Status distribute_part(td::uint32 seqno); + private: bool ready_ = false; @@ -205,6 +217,10 @@ class BroadcastFec : public td::ListNode { td::uint32 next_seqno_ = 0; td::uint64 received_parts_ = 0; + + std::map> parts_; + OverlayImpl *overlay_; + td::BufferSlice data_; }; class OverlayFecBroadcastPart : public td::ListNode { @@ -225,6 +241,7 @@ class OverlayFecBroadcastPart : public td::ListNode { td::BufferSlice signature_; bool is_short_; + bool untrusted_{false}; BroadcastFec *bcast_; OverlayImpl *overlay_; @@ -277,7 +294,7 @@ class OverlayFecBroadcastPart : public td::ListNode { signature_ = std::move(signature); } void update_overlay(OverlayImpl *overlay); - + tl_object_ptr export_tl(); tl_object_ptr export_tl_short(); td::BufferSlice export_serialized(); @@ -287,7 +304,9 @@ class OverlayFecBroadcastPart : public td::ListNode { td::Status run() { TRY_STATUS(run_checks()); TRY_STATUS(apply()); - TRY_STATUS(distribute()); + if(!untrusted_) { + TRY_STATUS(distribute()); + } return td::Status::OK(); } @@ -311,4 +330,3 @@ class OverlayFecBroadcastPart : public td::ListNode { } // namespace overlay } // namespace ton - diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index ae7c3dd5..1d133673 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -17,16 +17,22 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "overlay-manager.h" +#include "auto/tl/ton_api.h" #include "overlay.h" #include "adnl/utils.hpp" +#include "td/actor/actor.h" +#include "td/actor/common.h" #include "td/utils/Random.h" #include "td/db/RocksDb.h" +#include "td/utils/Status.h" #include "td/utils/overloaded.h" #include "keys/encryptor.h" +#include "td/utils/port/Poll.h" +#include namespace ton { @@ -268,17 +274,67 @@ void OverlayManager::save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort o db_.set(key, create_serialize_tl_object(std::move(obj))); } -Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature) +void OverlayManager::get_stats(td::Promise> promise) { + class Cb : public td::actor::Actor { + public: + Cb(td::Promise> promise) : promise_(std::move(promise)) { + } + void incr_pending() { + pending_++; + } + void decr_pending() { + if (!--pending_) { + promise_.set_result(create_tl_object(std::move(res_))); + stop(); + } + } + void receive_answer(tl_object_ptr res) { + if (res) { + res_.push_back(std::move(res)); + } + decr_pending(); + } + + private: + std::vector> res_; + size_t pending_{1}; + td::Promise> promise_; + }; + + auto act = td::actor::create_actor("overlaysstatsmerger", std::move(promise)).release(); + + for (auto &a : overlays_) { + for (auto &b : a.second) { + td::actor::send_closure(act, &Cb::incr_pending); + td::actor::send_closure(b.second, &Overlay::get_stats, + [act](td::Result> R) { + if (R.is_ok()) { + td::actor::send_closure(act, &Cb::receive_answer, R.move_as_ok()); + } else { + td::actor::send_closure(act, &Cb::receive_answer, nullptr); + } + }); + } + } + + td::actor::send_closure(act, &Cb::decr_pending); +} + +Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature) : issued_by_(issued_by) , expire_at_(expire_at) , max_size_(max_size) + , flags_(flags) , signature_(td::SharedSlice(signature.as_slice())) { } -Certificate::Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature) +Certificate::Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature) : issued_by_(issued_by) , expire_at_(expire_at) , max_size_(max_size) + , flags_(flags) , signature_(td::SharedSlice(signature.as_slice())) { } @@ -290,9 +346,19 @@ void Certificate::set_issuer(PublicKey issuer) { issued_by_ = issuer; } +constexpr td::uint32 cert_default_flags(td::uint32 max_size) { + return (max_size > Overlays::max_simple_broadcast_size() ? CertificateFlags::AllowFec : 0) | + CertificateFlags::Trusted; +} + td::BufferSlice Certificate::to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const { - return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, - max_size_); + if (flags_ == cert_default_flags(max_size_)) { + return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, + max_size_); + } else { + return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, + max_size_, flags_); + } } const PublicKeyHash Certificate::issuer_hash() const { @@ -307,32 +373,48 @@ const PublicKey &Certificate::issuer() const { td::Result> Certificate::create(tl_object_ptr cert) { std::shared_ptr res; - ton_api::downcast_call(*cert.get(), td::overloaded([&](ton_api::overlay_emptyCertificate &obj) { res = nullptr; }, - [&](ton_api::overlay_certificate &obj) { - res = std::make_shared( - PublicKey{obj.issued_by_}, obj.expire_at_, - static_cast(obj.max_size_), - std::move(obj.signature_)); - })); + ton_api::downcast_call(*cert.get(), + td::overloaded([&](ton_api::overlay_emptyCertificate &obj) { res = nullptr; }, + [&](ton_api::overlay_certificate &obj) { + res = std::make_shared(PublicKey{obj.issued_by_}, obj.expire_at_, + static_cast(obj.max_size_), + cert_default_flags(obj.max_size_), + std::move(obj.signature_)); + }, + [&](ton_api::overlay_certificateV2 &obj) { + res = std::make_shared(PublicKey{obj.issued_by_}, obj.expire_at_, + static_cast(obj.max_size_), + static_cast(obj.flags_), + std::move(obj.signature_)); + })); return std::move(res); } -td::Status Certificate::check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, - td::uint32 size) const { +BroadcastCheckResult Certificate::check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, + td::uint32 size, bool is_fec) const { if (size > max_size_) { - return td::Status::Error(ErrorCode::protoviolation, "too big broadcast size"); + return BroadcastCheckResult::Forbidden; } if (unix_time > expire_at_) { - return td::Status::Error(ErrorCode::protoviolation, "too old certificate"); + return BroadcastCheckResult::Forbidden; + } + if (is_fec && !(flags_ & CertificateFlags::AllowFec)) { + return BroadcastCheckResult::Forbidden; } - TRY_RESULT(E, issued_by_.get().create_encryptor()); + auto R1 = issued_by_.get().create_encryptor(); + if (R1.is_error()) { + return BroadcastCheckResult::Forbidden; + } + auto E = R1.move_as_ok(); auto B = to_sign(overlay_id, node); - TRY_STATUS(E->check_signature(B.as_slice(), signature_.as_slice())); + if (E->check_signature(B.as_slice(), signature_.as_slice()).is_error()) { + return BroadcastCheckResult::Forbidden; + } - return td::Status::OK(); + return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck; } tl_object_ptr Certificate::tl() const { diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index 574d0f91..c6afefaf 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -91,6 +91,7 @@ class OverlayManager : public Overlays { void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, td::actor::ActorOwn overlay); + void get_stats(td::Promise> promise) override; struct PrintId {}; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 47019c16..ec13fcd3 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -16,6 +16,7 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "auto/tl/ton_api.h" #include "td/utils/Random.h" #include "adnl/utils.hpp" @@ -25,6 +26,7 @@ #include "auto/tl/ton_api.hpp" #include "keys/encryptor.h" +#include "td/utils/StringBuilder.h" namespace ton { @@ -391,25 +393,21 @@ td::Status OverlayImpl::check_date(td::uint32 date) { return td::Status::OK(); } -td::Status OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size) { +BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, + bool is_fec) { if (size == 0) { - return td::Status::Error(ErrorCode::protoviolation, "empty broadcast"); + return BroadcastCheckResult::Forbidden; } auto short_id = source.compute_short_id(); - auto r = rules_.max_size(source.compute_short_id()); - if (r >= size) { - return td::Status::OK(); + auto r = rules_.check_rules(source.compute_short_id(), size, is_fec); + if (!cert || r == BroadcastCheckResult::Allowed) { + return r; } - if (!cert) { - return td::Status::Error(ErrorCode::protoviolation, "source is not eligible"); - } - TRY_STATUS(cert->check(short_id, overlay_id_, static_cast(td::Clocks::system()), size)); - auto issuer_short = cert->issuer_hash(); - if (rules_.max_size(issuer_short) < size) { - return td::Status::Error(ErrorCode::protoviolation, "bad certificate"); - } - return td::Status::OK(); + + auto r2 = cert->check(short_id, overlay_id_, static_cast(td::Clocks::system()), size, is_fec); + r2 = broadcast_check_result_min(r2, rules_.check_rules(cert->issuer_hash(), size, is_fec)); + return broadcast_check_result_max(r, r2); } td::Status OverlayImpl::check_delivered(BroadcastHash hash) { @@ -539,6 +537,38 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) { rules_ = std::move(rules); } +void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { + callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); +} + +void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result R) { + { + auto it = broadcasts_.find(hash); + if (it != broadcasts_.end()) { + it->second->broadcast_checked(std::move(R)); + } + } + { + auto it = fec_broadcasts_.find(hash); + if (it != fec_broadcasts_.end()) { + it->second->broadcast_checked(std::move(R)); + } + } +} + +void OverlayImpl::get_stats(td::Promise> promise) { + auto res = create_tl_object(); + res->adnl_id_ = local_id_.bits256_value(); + res->overlay_id_ = overlay_id_.bits256_value(); + res->overlay_id_full_ = id_full_.pubkey().tl(); + peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { res->nodes_.push_back(key.tl()); }); + + res->stats_.push_back( + create_tl_object("neighbours_cnt", PSTRING() << neighbours_.size())); + + promise.set_value(std::move(res)); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.h b/overlay/overlay.h index cba15145..cce24ed8 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -63,6 +63,7 @@ class Overlay : public td::actor::Actor { virtual void add_certificate(PublicKeyHash key, std::shared_ptr) = 0; virtual void set_privacy_rules(OverlayPrivacyRules rules) = 0; virtual void receive_nodes_from_db(tl_object_ptr nodes) = 0; + virtual void get_stats(td::Promise> promise) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; }; @@ -70,4 +71,3 @@ class Overlay : public td::actor::Actor { } // namespace overlay } // namespace ton - diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index b087fc75..76d17aab 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -144,9 +144,12 @@ class OverlayImpl : public Overlay { void print(td::StringBuilder &sb) override; td::Status check_date(td::uint32 date); - td::Status check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size); + BroadcastCheckResult check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec); td::Status check_delivered(BroadcastHash hash); + void broadcast_checked(Overlay::BroadcastHash hash, td::Result R); + void check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise); + BroadcastFec *get_fec_broadcast(BroadcastHash hash); void register_fec_broadcast(std::unique_ptr bcast); void register_simple_broadcast(std::unique_ptr bcast); @@ -187,6 +190,8 @@ class OverlayImpl : public Overlay { std::shared_ptr get_certificate(PublicKeyHash local_id); td::Result get_encryptor(PublicKey source); + void get_stats(td::Promise> promise) override; + private: template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { diff --git a/overlay/overlays.h b/overlay/overlays.h index 90f3877e..799faf5a 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -21,7 +21,11 @@ #include "adnl/adnl.h" #include "dht/dht.h" +#include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" #include @@ -80,41 +84,65 @@ class OverlayIdFull { td::BufferSlice name_; }; +struct CertificateFlags { + enum Values : td::uint32 { AllowFec = 1, Trusted = 2 }; +}; + +enum BroadcastCheckResult { Forbidden = 1, NeedCheck = 2, Allowed = 3 }; + +inline BroadcastCheckResult broadcast_check_result_max(BroadcastCheckResult l, BroadcastCheckResult r) { + return static_cast(std::max(static_cast(l), static_cast(r))); +} +inline BroadcastCheckResult broadcast_check_result_min(BroadcastCheckResult l, BroadcastCheckResult r) { + return static_cast(std::min(static_cast(l), static_cast(r))); +} + class OverlayPrivacyRules { public: OverlayPrivacyRules() { } OverlayPrivacyRules(td::uint32 size) : max_unath_size_(size) { } - OverlayPrivacyRules(td::uint32 max_size, std::map authorized_keys) - : max_unath_size_(max_size), authorized_keys_(std::move(authorized_keys)) { + OverlayPrivacyRules(td::uint32 max_size, td::uint32 flags, std::map authorized_keys) + : max_unath_size_(max_size), flags_(flags), authorized_keys_(std::move(authorized_keys)) { } - td::uint32 max_size(PublicKeyHash hash) { + BroadcastCheckResult check_rules(PublicKeyHash hash, td::uint32 size, bool is_fec) { + auto it = authorized_keys_.find(hash); if (it == authorized_keys_.end()) { - return max_unath_size_; + if (size > max_unath_size_) { + return BroadcastCheckResult::Forbidden; + } + if (!(flags_ & CertificateFlags::AllowFec) && is_fec) { + return BroadcastCheckResult::Forbidden; + } + return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck; } else { - return it->second; + return it->second >= size ? BroadcastCheckResult::Allowed : BroadcastCheckResult::Forbidden; } } private: td::uint32 max_unath_size_{0}; + td::uint32 flags_{0}; std::map authorized_keys_; }; class Certificate { public: - Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature); - Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature); + Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature); + Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature); Certificate() { } void set_signature(td::BufferSlice signature); void set_issuer(PublicKey issuer); td::BufferSlice to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const; - td::Status check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size) const; + BroadcastCheckResult check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size, + bool is_fec) const; tl_object_ptr tl() const; const PublicKey &issuer() const; const PublicKeyHash issuer_hash() const; @@ -126,6 +154,7 @@ class Certificate { td::Variant issued_by_; td::int32 expire_at_; td::uint32 max_size_; + td::uint32 flags_; td::SharedSlice signature_; }; @@ -137,6 +166,10 @@ class Overlays : public td::actor::Actor { virtual void receive_query(adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, td::BufferSlice data, td::Promise promise) = 0; virtual void receive_broadcast(PublicKeyHash src, OverlayIdShort overlay_id, td::BufferSlice data) = 0; + virtual void check_broadcast(PublicKeyHash src, OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) { + promise.set_value(td::Unit()); + } virtual ~Callback() = default; }; @@ -199,6 +232,7 @@ class Overlays : public td::actor::Actor { virtual void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) = 0; + virtual void get_stats(td::Promise> promise) = 0; }; } // namespace overlay diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index f93f6e1e..33ae5b67 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -51,7 +51,7 @@ target_include_directories(rldp PUBLIC ${OPENSSL_INCLUDE_DIR} ) if (GSL_FOUND) - target_link_libraries(rldp2 PRIVATE GSL::gsl) + target_link_libraries(rldp2 PRIVATE gsl) target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1) endif() target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api) diff --git a/tdutils/td/utils/DecTree.h b/tdutils/td/utils/DecTree.h index 80bd0e21..1ebb6cd6 100644 --- a/tdutils/td/utils/DecTree.h +++ b/tdutils/td/utils/DecTree.h @@ -46,6 +46,17 @@ class DecTree { } } + template + void iterate(const FuncT &cb) { + if (left_) { + left_->iterate(cb); + } + cb(key_, value_); + if (right_) { + right_->iterate(cb); + } + } + Node(KeyType key, ValueType value, uint32 y) : size_(1), key_(std::move(key)), value_(std::move(value)), y_(y) { } }; @@ -223,6 +234,15 @@ class DecTree { bool exists(const KeyType &key) const { return get_node(root_, key) != nullptr; } + + template + void iterate(const FuncT &cb) { + if (size() == 0) { + return; + } else { + root_->iterate(cb); + } + } }; } // namespace td diff --git a/third-party/rocksdb b/third-party/rocksdb index 0915c99f..fcf3d75f 160000 --- a/third-party/rocksdb +++ b/third-party/rocksdb @@ -1 +1 @@ -Subproject commit 0915c99f01b46f50af8e02da8b6528156f584b7c +Subproject commit fcf3d75f3f022a6a55ff1222d6b06f8518d38c7c diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index 7d4495e7..d31ef569 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -25,6 +25,7 @@ adnl.message.answer query_id:int256 answer:bytes = adnl.Message; liteServer.error code:int message:string = liteServer.Error; liteServer.accountId workchain:int id:int256 = liteServer.AccountId; +liteServer.libraryEntry hash:int256 data:bytes = liteServer.LibraryEntry; liteServer.masterchainInfo last:tonNode.blockIdExt state_root_hash:int256 init:tonNode.zeroStateIdExt = liteServer.MasterchainInfo; liteServer.masterchainInfoExt mode:# version:int capabilities:long last:tonNode.blockIdExt last_utime:int now:int state_root_hash:int256 init:tonNode.zeroStateIdExt = liteServer.MasterchainInfoExt; @@ -50,6 +51,7 @@ liteServer.blockLinkForward to_key_block:Bool from:tonNode.blockIdExt to:tonNode liteServer.partialBlockProof complete:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt steps:(vector liteServer.BlockLink) = liteServer.PartialBlockProof; liteServer.configInfo mode:# id:tonNode.blockIdExt state_proof:bytes config_proof:bytes = liteServer.ConfigInfo; liteServer.validatorStats mode:# id:tonNode.blockIdExt count:int complete:Bool state_proof:bytes data_proof:bytes = liteServer.ValidatorStats; +liteServer.libraryResult result:(vector liteServer.libraryEntry) = liteServer.LibraryResult; liteServer.debug.verbosity value:int = liteServer.debug.Verbosity; @@ -75,6 +77,7 @@ liteServer.getBlockProof mode:# known_block:tonNode.blockIdExt target_block:mode liteServer.getConfigAll mode:# id:tonNode.blockIdExt = liteServer.ConfigInfo; liteServer.getConfigParams mode:# id:tonNode.blockIdExt param_list:(vector int) = liteServer.ConfigInfo; liteServer.getValidatorStats#091a58bc mode:# id:tonNode.blockIdExt limit:int start_after:mode.0?int256 modified_after:mode.2?int = liteServer.ValidatorStats; +liteServer.getLibraries library_list:(vector int256) = liteServer.LibraryResult; liteServer.queryPrefix = Object; liteServer.query data:bytes = Object; diff --git a/tl/generate/scheme/lite_api.tlo b/tl/generate/scheme/lite_api.tlo index d524f54b..eb8e3211 100644 Binary files a/tl/generate/scheme/lite_api.tlo and b/tl/generate/scheme/lite_api.tlo differ diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 57ed9ddc..0be70540 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -219,9 +219,11 @@ overlay.broadcastFec.partId broadcast_hash:int256 data_hash:int256 seqno:int = o overlay.broadcast.toSign hash:int256 date:int = overlay.broadcast.ToSign; overlay.certificate issued_by:PublicKey expire_at:int max_size:int signature:bytes = overlay.Certificate; +overlay.certificateV2 issued_by:PublicKey expire_at:int max_size:int flags:int signature:bytes = overlay.Certificate; overlay.emptyCertificate = overlay.Certificate; overlay.certificateId overlay_id:int256 node:int256 expire_at:int max_size:int = overlay.CertificateId; +overlay.certificateIdV2 overlay_id:int256 node:int256 expire_at:int max_size:int flags:int = overlay.CertificateId; overlay.unicast data:bytes = overlay.Broadcast; overlay.broadcast src:PublicKey certificate:overlay.Certificate flags:int data:bytes date:int signature:bytes = overlay.Broadcast; @@ -618,6 +620,11 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus; engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus; +engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 nodes:(vector adnl.id.short) + stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; +engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; + + ---functions--- engine.validator.getTime = engine.validator.Time; @@ -659,8 +666,14 @@ engine.validator.createComplaintVote election_id:int vote:bytes = engine.validat engine.validator.checkDhtServers id:int256 = engine.validator.DhtServersStatus; +engine.validator.getOverlaysStats = engine.validator.OverlaysStats; + engine.validator.controlQuery data:bytes = Object; +engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; +engine.validator.signShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash expire_at:int max_size:int = overlay.Certificate; +engine.validator.importShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 823219b5..dbdd73b3 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index b1dd7679..d5f97cce 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -208,8 +208,11 @@ blocks.shards shards:vector = blocks.Shards; blocks.accountTransactionId account:bytes lt:int64 = blocks.AccountTransactionId; blocks.shortTxId mode:# account:mode.0?bytes lt:mode.1?int64 hash:mode.2?bytes = liteServer.TransactionId; blocks.transactions id:ton.blockIdExt req_count:int32 incomplete:Bool transactions:vector = blocks.Transactions; -blocks.header id:ton.blockIdExt global_id:int32 version:int32 after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 vert_seqno:# prev_blocks:vector = blocks.Header; +blocks.header id:ton.blockIdExt global_id:int32 version:int32 flags:# after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 gen_utime:int53 vert_seqno:# prev_blocks:vector = blocks.Header; //blocks.shortData header:blocks.Header transactions:blocks.Header = blocks.BlockData; + +configInfo config:tvm.cell = ConfigInfo; + ---functions--- init options:options = options.Info; @@ -262,6 +265,8 @@ guessAccount public_key:string rwallet_init_public_key:string = AccountRevisionL getAccountState account_address:accountAddress = FullAccountState; createQuery private_key:InputKey address:accountAddress timeout:int32 action:Action initial_account_state:InitialAccountState = query.Info; +getConfigParam mode:# id:ton.blockIdExt param:# = ConfigInfo; + msg.decrypt input_key:InputKey data:msg.dataEncryptedArray = msg.DataDecryptedArray; msg.decryptWithProof proof:bytes data:msg.dataEncrypted = msg.Data; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 77741950..05c56ac0 100644 Binary files a/tl/generate/scheme/tonlib_api.tlo and b/tl/generate/scheme/tonlib_api.tlo differ diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index d07482ce..0111095b 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -62,9 +62,7 @@ void LastConfig::with_last_block(td::Result r_last_block) { } auto last_block = r_last_block.move_as_ok(); - auto params = params_; - client_.send_query(ton::lite_api::liteServer_getConfigParams(0, create_tl_lite_block_id(last_block.last_block_id), - std::move(params)), + client_.send_query(ton::lite_api::liteServer_getConfigAll(0, create_tl_lite_block_id(last_block.last_block_id)), [this](auto r_config) { this->on_config(std::move(r_config)); }); } diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 1ac3a0d7..2696bb1a 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -861,13 +861,13 @@ class Query { } return res; } - td::Result>> estimate_fees(bool ignore_chksig, const block::Config& cfg) { + td::Result>> estimate_fees(bool ignore_chksig, std::shared_ptr& cfg, vm::Dictionary& libraries) { // gas fees bool is_masterchain = raw_.source->get_address().workchain == ton::masterchainId; - TRY_RESULT(gas_limits_prices, cfg.get_gas_limits_prices(is_masterchain)); - TRY_RESULT(storage_prices, cfg.get_storage_prices()); - TRY_RESULT(masterchain_msg_prices, cfg.get_msg_prices(true)); - TRY_RESULT(basechain_msg_prices, cfg.get_msg_prices(false)); + TRY_RESULT(gas_limits_prices, cfg->get_gas_limits_prices(is_masterchain)); + TRY_RESULT(storage_prices, cfg->get_storage_prices()); + TRY_RESULT(masterchain_msg_prices, cfg->get_msg_prices(true)); + TRY_RESULT(basechain_msg_prices, cfg->get_msg_prices(false)); block::MsgPrices* msg_prices[2] = {&basechain_msg_prices, &masterchain_msg_prices}; auto storage_fee_256 = block::StoragePrices::compute_storage_fees( raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_stat, @@ -888,7 +888,9 @@ class Query { .set_limits(gas_limits) .set_balance(raw_.source->get_balance()) .set_now(raw_.source->get_sync_time()) - .set_ignore_chksig(ignore_chksig)); + .set_ignore_chksig(ignore_chksig) + .set_address(raw_.source->get_address()) + .set_config(cfg).set_libraries(libraries)); td::int64 fwd_fee = 0; if (res.success) { LOG(DEBUG) << "output actions:\n" @@ -910,7 +912,7 @@ class Query { for (auto& destination : raw_.destinations) { bool dest_is_masterchain = destination && destination->get_address().workchain == ton::masterchainId; - TRY_RESULT(dest_gas_limits_prices, cfg.get_gas_limits_prices(dest_is_masterchain)); + TRY_RESULT(dest_gas_limits_prices, cfg->get_gas_limits_prices(dest_is_masterchain)); auto dest_storage_fee_256 = destination ? block::StoragePrices::compute_storage_fees( destination->get_sync_time(), storage_prices, destination->raw().storage_stat, @@ -1887,6 +1889,8 @@ td::Status TonlibClient::do_request(const tonlib_api::init& request, TRY_RESULT(kv, std::move(r_kv)); kv_ = std::shared_ptr(kv.release()); + load_libs_from_disk(); + key_storage_.set_key_value(kv_); last_block_storage_.set_key_value(kv_); auto res = tonlib_api::make_object(); @@ -2306,9 +2310,10 @@ struct ToRawTransactions { return td::Status::Error("Failed to unpack CommonMsgInfo::ext_out_msg_info"); } TRY_RESULT(src, to_std_address(msg_info.src)); + auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( tonlib_api::make_object(src), - tonlib_api::make_object(), 0, 0, 0, 0, std::move(body_hash), get_data(src)); + tonlib_api::make_object(), 0, 0, 0, created_lt, std::move(body_hash), get_data(src)); } } @@ -2353,7 +2358,7 @@ struct ToRawTransactions { if (trans.outmsg_cnt != 0) { vm::Dictionary dict{trans.r1.out_msgs, 15}; - for (int x = 0; x < trans.outmsg_cnt && x < 100; x++) { + for (int x = 0; x < trans.outmsg_cnt; x++) { TRY_RESULT(out_msg, to_raw_message(dict.lookup_ref(td::BitArray<15>{x}))); fees += out_msg->fwd_fee_; fees += out_msg->ihr_fee_; @@ -3266,7 +3271,7 @@ void TonlibClient::query_estimate_fees(td::int64 id, bool ignore_chksig, td::Res return; } TRY_RESULT_PROMISE(promise, state, std::move(r_state)); - TRY_RESULT_PROMISE_PREFIX(promise, fees, TRY_VM(it->second->estimate_fees(ignore_chksig, *state.config)), + TRY_RESULT_PROMISE_PREFIX(promise, fees, TRY_VM(it->second->estimate_fees(ignore_chksig, state.config, libraries)), TonlibError::Internal()); promise.set_value(tonlib_api::make_object( fees.first.to_tonlib_api(), td::transform(fees.second, [](auto& x) { return x.to_tonlib_api(); }))); @@ -3440,7 +3445,8 @@ td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { [&](tonlib_api::tvm_stackEntryUnsupported& cell) { return td::Status::Error("Unsuppored stack entry"); }, [&](tonlib_api::tvm_stackEntrySlice& cell) -> td::Result { TRY_RESULT(res, vm::std_boc_deserialize(cell.slice_->bytes_)); - return vm::StackEntry{std::move(res)}; + auto slice = vm::load_cell_slice_ref(std::move(res)); + return vm::StackEntry{std::move(slice)}; }, [&](tonlib_api::tvm_stackEntryCell& cell) -> td::Result { TRY_RESULT(res, vm::std_boc_deserialize(cell.cell_->bytes_)); @@ -3472,6 +3478,38 @@ td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { })); } +void deep_library_search(std::set& set, std::set& visited, + vm::Dictionary libs, td::Ref cell, int depth) { + if (depth <= 0 || set.size() >= 16 || visited.size() >= 256) { + return; + } + auto ins = visited.insert(cell->get_hash()); + if (!ins.second) { + return; // already visited this cell + } + auto r_loaded_cell = cell->load_cell(); + if (r_loaded_cell.is_error()) { + return; + } + auto loaded_cell = r_loaded_cell.move_as_ok(); + if (loaded_cell.data_cell->is_special()) { + if (loaded_cell.data_cell->special_type() == vm::DataCell::SpecialType::Library) { + vm::CellSlice cs(std::move(loaded_cell)); + if (cs.size() != vm::Cell::hash_bits + 8) { + return; + } + auto key = td::Bits256(cs.data_bits() + 8); + if (libs.lookup(key).is_null()) { + set.insert(key); + } + } + return; + } + for (unsigned int i=0; iget_refs_cnt(); i++) { + deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1); + } +} + td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, td::Promise>&& promise) { auto it = smcs_.find(request.id_); @@ -3493,15 +3531,120 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, args.set_stack(std::move(stack)); args.set_balance(it->second->get_balance()); args.set_now(it->second->get_sync_time()); - auto res = smc->run_get_method(std::move(args)); + args.set_address(it->second->get_address()); + + client_.with_last_config([self = this, smc = std::move(smc), args = std::move(args), promise = std::move(promise) + ](td::Result r_state) mutable { + TRY_RESULT_PROMISE(promise, state, std::move(r_state)); + args.set_config(state.config); + + auto code = smc->get_state().code; + if (code.not_null()) { + std::set librarySet; + std::set visited; + deep_library_search(librarySet, visited, self->libraries, code, 24); + std::vector libraryList{librarySet.begin(), librarySet.end()}; + if (libraryList.size() > 0) { + LOG(DEBUG) << "Requesting found libraries in code (" << libraryList.size() << ")"; + self->client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(libraryList)), + [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)] + (td::Result> r_libraries) mutable + { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); + } else { + auto libraries = r_libraries.move_as_ok(); + bool updated = false; + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + updated = true; + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + self->store_libs_to_disk(); + } + } + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + }); + } + else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + } + } + else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + } + }); + return td::Status::OK(); +} + +void TonlibClient::perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, + td::Promise>&& promise) { + + args.set_libraries(libraries); + + auto res = smc->run_get_method(args); // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; std::vector> res_stack; for (auto& entry : res.stack->as_span()) { res_stack.push_back(to_tonlib_api(entry)); } - promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); - return td::Status::OK(); + + if (res.missing_library.not_null()) { + td::Bits256 hash = res.missing_library; + LOG(DEBUG) << "Requesting missing library: " << hash.to_hex(); + std::vector req = {std::move(hash)}; + client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(req)), + [self = this, res = std::move(res), res_stack = std::move(res_stack), hash = std::move(hash), + smc = std::move(smc), args = std::move(args), promise = std::move(promise)] + (td::Result> r_libraries) mutable + { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain missing library: " << r_libraries.move_as_error().to_string(); + promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + return; + } + bool found = false, updated = false; + auto libraries = r_libraries.move_as_ok(); + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + found |= (lr->hash_ == hash); + updated = true; + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + self->store_libs_to_disk(); + } + if (!found) { + LOG(WARNING) << "cannot obtain library " << hash.to_hex() << ", it may not exist"; + promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + } else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + } + }); + } + else { + promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + } } td::Result> to_tonlib_api( @@ -4057,6 +4200,32 @@ auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result>&& promise) { + TRY_RESULT(lite_block, to_lite_api(*request.id_)) + auto block = create_block_id(std::move(lite_block)); + auto param = request.param_; + std::vector params = { param }; + + client_.send_query(ton::lite_api::liteServer_getConfigParams(0, std::move(lite_block), std::move(params)), + promise.wrap([block, param](auto r_config) { + auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), + r_config->config_proof_.as_slice()); + if (state.is_error()) { + LOG(ERROR) << "block::check_extract_state_proof failed: " << state.error(); + } + auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); + if (config.is_error()) { + LOG(ERROR) << "block::Config::extract_from_state failed: " << config.error(); + } + tonlib_api::configInfo config_result; + config_result.config_ = tonlib_api::make_object(to_bytes(config.move_as_ok()->get_config_param(param))); + return tonlib_api::make_object(std::move(config_result)); + })); + + return td::Status::OK(); +} + td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainInfo& masterchain_info, td::Promise>&& promise) { client_.send_query(ton::lite_api::liteServer_getMasterchainInfo(), @@ -4190,6 +4359,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req header.id_ = to_tonlib_api(blk_id); header.global_id_ = blk.global_id; header.version_ = info.version; + header.flags_ = info.flags; header.after_merge_ = info.after_merge; header.after_split_ = info.after_split; header.before_split_ = info.before_split; @@ -4200,6 +4370,8 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req header.min_ref_mc_seqno_ = info.min_ref_mc_seqno; header.start_lt_ = info.start_lt; header.end_lt_ = info.end_lt; + header.gen_utime_ = info.gen_utime; + header.is_key_block_ = info.key_block; header.vert_seqno_ = info.vert_seq_no; if(!info.not_master) { header.prev_key_block_seqno_ = info.prev_key_block_seqno; @@ -4229,6 +4401,28 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req return td::Status::OK(); } +void TonlibClient::load_libs_from_disk() { + LOG(DEBUG) << "loading libraries from disk cache"; + auto r_data = kv_->get("tonlib.libcache"); + if (r_data.is_error()) { + return; + } + auto r_dict = vm::std_boc_deserialize(r_data.move_as_ok(), true); + if (r_dict.is_error()) { + return; + } + libraries = vm::Dictionary(vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice( + r_dict.move_as_ok())).finalize()), 256); + // int n = 0; for (auto&& lr : libraries) n++; + LOG(DEBUG) << "loaded libraries from disk cache"; +} + +void TonlibClient::store_libs_to_disk() { // NB: Dictionary.get_root_cell does not compute_root, and it is protected + kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root()) + .finalize()).move_as_ok().as_slice()); + // int n = 0; for (auto&& lr : libraries) n++; + LOG(DEBUG) << "stored libraries to disk cache"; +} template td::Status TonlibClient::do_request(const tonlib_api::runTests& request, P&&) { diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index bedc1d49..716133e4 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -107,6 +107,7 @@ class TonlibClient : public td::actor::Actor { td::optional block_id; }; QueryContext query_context_; + vm::Dictionary libraries{256}; // network td::actor::ActorOwn raw_client_; @@ -321,7 +322,8 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(tonlib_api::pchan_unpackPromise& request, td::Promise>&& promise); - + void perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, + td::Promise>&& promise); void do_dns_request(std::string name, td::int32 category, td::int32 ttl, td::optional block_id, block::StdAddress address, td::Promise>&& promise); @@ -355,8 +357,14 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::blocks_getBlockHeader& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::getConfigParam& request, + td::Promise>&& promise); + void proxy_request(td::int64 query_id, std::string data); + void load_libs_from_disk(); + void store_libs_to_disk(); + friend class TonlibQueryActor; struct Target { bool can_be_empty{true}; diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 64f57de1..4b5802c6 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -26,9 +26,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "validator-engine-console-query.h" +#include "auto/tl/ton_api.h" +#include "td/utils/StringBuilder.h" #include "validator-engine-console.h" #include "terminal/terminal.h" #include "td/utils/filesystem.h" +#include "overlay/overlays.h" #include @@ -720,3 +723,193 @@ td::Status CheckDhtServersQuery::receive(td::BufferSlice data) { } return td::Status::OK(); } + +td::Status SignCertificateQuery::run() { + TRY_RESULT_ASSIGN(overlay_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(signer_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(out_file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status SignCertificateQuery::send() { + auto cid = ton::create_serialize_tl_object(overlay_, id_, expire_at_, max_size_); + auto sign = ton::create_serialize_tl_object(signer_.tl(), std::move(cid)); + auto pub = ton::create_serialize_tl_object(signer_.tl()); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(pub), + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); + } + })); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(sign), + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); + } + })); + return td::Status::OK(); +} + +void SignCertificateQuery::receive_pubkey(td::BufferSlice R) { + auto f = ton::fetch_tl_object(R.as_slice(), true); + if (f.is_error()) { + handle_error(f.move_as_error_prefix("Failed to get pubkey: ")); + return; + } + pubkey_ = f.move_as_ok(); + has_pubkey_ = true; + if(has_signature_) { + save_certificate(); + } +} + + +td::Status SignCertificateQuery::receive(td::BufferSlice data) { + UNREACHABLE(); +} + +void SignCertificateQuery::receive_signature(td::BufferSlice R) { + auto f = ton::fetch_tl_object(R.as_slice(), true); + if(f.is_error()){ + handle_error(f.move_as_error_prefix("Failed to get signature: ")); + return; + } + signature_ = std::move(f.move_as_ok()->signature_); + if(has_pubkey_) { + save_certificate(); + } +} + +void SignCertificateQuery::save_certificate() { + auto c = ton::create_serialize_tl_object( + std::move(pubkey_), expire_at_, max_size_, std::move(signature_)); + auto w = td::write_file(out_file_, c.as_slice()); + if(w.is_error()) { + handle_error(w.move_as_error_prefix("Failed to write certificate to file: ")); + return; + } + td::TerminalIO::out() << "saved certificate\n"; + stop(); +} + +td::Status ImportCertificateQuery::run() { + TRY_RESULT_ASSIGN(overlay_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(kh_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status ImportCertificateQuery::send() { + TRY_RESULT(data, td::read_file(in_file_)); + TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), + "incorrect certificate"); + auto b = ton::create_serialize_tl_object( + overlay_, + ton::create_tl_object(id_), + ton::create_tl_object(kh_.tl()), + std::move(cert) + ); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} +td::Status GetOverlaysStatsQuery::run() { + return td::Status::OK(); +} + +td::Status GetOverlaysStatsQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + for (auto &s : f->overlays_) { + td::StringBuilder sb; + sb << "overlay_id=" << s->overlay_id_ << " adnl_id=" << s->adnl_id_ << "\n"; + sb << " nodes:"; + for (auto &n : s->nodes_) { + sb << " " << n->id_ << "\n"; + } + sb << " stats:\n"; + for (auto &t : s->stats_) { + sb << " " << t->key_ << "\t" << t->value_ << "\n"; + } + td::TerminalIO::output(sb.as_cslice()); + } + return td::Status::OK(); +} + + +td::Status ImportCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; + return td::Status::OK(); +} + + +td::Status SignShardOverlayCertificateQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(out_file_, tokenizer_.get_token()); + + return td::Status::OK(); +} + +td::Status SignShardOverlayCertificateQuery::send() { + auto b = ton::create_serialize_tl_object + (wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(c, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect cert: "); + auto w = td::write_file(out_file_, data.as_slice()); + if(w.is_error()) { + return w.move_as_error_prefix("Failed to write certificate to file: "); + } + td::TerminalIO::out() << "saved certificate\n"; + + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); + + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::send() { + TRY_RESULT(data, td::read_file(in_file_)); + TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), + "incorrect certificate"); + auto b = ton::create_serialize_tl_object + (wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index e010d76f..769f2052 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -903,3 +903,139 @@ class CheckDhtServersQuery : public Query { private: ton::PublicKeyHash id_; }; + +class GetOverlaysStatsQuery : public Query { + public: + GetOverlaysStatsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getoverlaysstats"; + } + static std::string get_help() { + return "getoverlaysstats\tgets stats for all overlays"; + } + std::string name() const override { + return get_name(); + } +}; + +class SignCertificateQuery : public Query { + public: + SignCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "signcert"; + } + static std::string get_help() { + return "signcert \tsign overlay certificate by key"; + } + std::string name() const override { + return get_name(); + } + void receive_pubkey(td::BufferSlice R); + void receive_signature(td::BufferSlice R); + + + private: + void save_certificate(); + + td::Bits256 overlay_; + td::Bits256 id_; + td::int32 expire_at_; + td::uint32 max_size_; + std::string out_file_; + ton::PublicKeyHash signer_; + td::BufferSlice signature_; + std::unique_ptr pubkey_; + bool has_signature_{0}; + bool has_pubkey_{0}; +}; + +class ImportCertificateQuery : public Query { + public: + ImportCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importcert"; + } + static std::string get_help() { + return "importcert \timport overlay certificate for specific key"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 overlay_; + td::Bits256 id_; + ton::PublicKeyHash kh_; + std::string in_file_; +}; + +class SignShardOverlayCertificateQuery : public Query { + public: + SignShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "signshardoverlaycert"; + } + static std::string get_help() { + return "signshardoverlaycert \tsign certificate for in currently active shard overlay"; + } + std::string name() const override { + return get_name(); + } + + private: + + td::int32 wc_; + td::int64 shard_; + td::int32 expire_at_; + ton::PublicKeyHash key_; + td::uint32 max_size_; + std::string out_file_; +}; + + +class ImportShardOverlayCertificateQuery : public Query { + public: + ImportShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importshardoverlaycert"; + } + static std::string get_help() { + return "importshardoverlaycert \timport certificate for in currently active shard overlay"; + } + std::string name() const override { + return get_name(); + } + + private: + + td::int32 wc_; + td::int64 shard_; + ton::PublicKeyHash key_; + std::string in_file_; +}; + diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 43a02dc6..899cd53f 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -134,6 +134,11 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { @@ -258,7 +263,8 @@ int main(int argc, char* argv[]) { std::exit(2); }); p.add_option('V', "version", "shows validator-engine-console build information", [&]() { - std::cout << "validator-engine-console build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "validator-engine-console build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_checked_option('a', "address", "server address", [&](td::Slice arg) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 16f71d5d..59279bef 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -27,6 +27,11 @@ */ #include "validator-engine.hpp" +#include "auto/tl/ton_api.h" +#include "overlay-manager.h" +#include "td/actor/actor.h" +#include "tl-utils/tl-utils.hpp" +#include "tl/TlObject.h" #include "ton/ton-types.h" #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" @@ -1309,6 +1314,9 @@ td::Status ValidatorEngine::load_global_config() { if (state_ttl_ != 0) { validator_options_.write().set_state_ttl(state_ttl_); } + if (max_mempool_num_ != 0) { + validator_options_.write().set_max_mempool_num(max_mempool_num_); + } if (block_ttl_ != 0) { validator_options_.write().set_block_ttl(block_ttl_); } @@ -3140,6 +3148,135 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_createCom .release(); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r = ton::overlay::Certificate::create(std::move(query.cert_)); + if(r.is_error()) { + promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); + } + //TODO force Overlays::update_certificate to return result + /*auto P = td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); + */ + td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::update_certificate, + ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, + ton::overlay::OverlayIdShort{query.overlay_id_}, + ton::PublicKeyHash{query.signed_key_->key_hash_}, + r.move_as_ok()); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true) + ); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r = ton::overlay::Certificate::create(std::move(query.cert_)); + if(r.is_error()) { + promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); + } + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to import cert: "))); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); + ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, + shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to import cert: "))); + } else { + promise.set_value(R.move_as_ok()); + } + }); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, + shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(P)); +} + + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::get_stats, + [promise = std::move(promise)]( + td::Result> R) mutable { + if (R.is_ok()) { + promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); + } else { + promise.set_value(create_control_query_error( + td::Status::Error(ton::ErrorCode::notready, "overlay manager not ready"))); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -3291,7 +3428,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows validator-engine build information", [&]() { - std::cout << "validator-engine build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "validator-engine build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { @@ -3336,6 +3474,10 @@ int main(int argc, char *argv[]) { auto v = td::to_double(fname); acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); }); }); + p.add_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) { + auto v = td::to_double(fname); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); + }); p.add_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", [&](td::Slice fname) { auto v = td::to_double(fname); @@ -3367,18 +3509,20 @@ int main(int argc, char *argv[]) { acts.push_back([&x, seq]() { td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain, seq); }); return td::Status::OK(); }); - p.add_checked_option( - 'F', "unsafe-catchain-rotate", "use forceful and DANGEROUS catchain rotation", [&](td::Slice params) { - auto pos1 = params.find(':'); - TRY_RESULT(b_seq, td::to_integer_safe(params.substr(0, pos1))); - params = params.substr(++pos1, params.size()); - auto pos2 = params.find(':'); - TRY_RESULT(cc_seq, td::to_integer_safe(params.substr(0, pos2))); - params = params.substr(++pos2, params.size()); - auto h = std::stoi(params.substr(0, params.size()).str()); - acts.push_back([&x, b_seq, cc_seq, h]() { td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain_rotation, b_seq, cc_seq, h); }); - return td::Status::OK(); - }); + p.add_checked_option('F', "unsafe-catchain-rotate", "use forceful and DANGEROUS catchain rotation", + [&](td::Slice params) { + auto pos1 = params.find(':'); + TRY_RESULT(b_seq, td::to_integer_safe(params.substr(0, pos1))); + params = params.substr(++pos1, params.size()); + auto pos2 = params.find(':'); + TRY_RESULT(cc_seq, td::to_integer_safe(params.substr(0, pos2))); + params = params.substr(++pos2, params.size()); + auto h = std::stoi(params.substr(0, params.size()).str()); + acts.push_back([&x, b_seq, cc_seq, h]() { + td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain_rotation, b_seq, cc_seq, h); + }); + return td::Status::OK(); + }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 8ce56ab4..88ba802d 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -28,6 +28,7 @@ #pragma once #include "adnl/adnl.h" +#include "auto/tl/ton_api.h" #include "rldp/rldp.h" #include "dht/dht.h" #include "validator/manager.h" @@ -191,6 +192,7 @@ class ValidatorEngine : public td::actor::Actor { std::map control_permissions_; double state_ttl_ = 0; + double max_mempool_num_ = 0; double block_ttl_ = 0; double sync_ttl_ = 0; double archive_ttl_ = 0; @@ -223,6 +225,9 @@ class ValidatorEngine : public td::actor::Actor { void set_state_ttl(double t) { state_ttl_ = t; } + void set_max_mempool_num(double t) { + max_mempool_num_ = t; + } void set_block_ttl(double t) { block_ttl_ = t; } @@ -391,6 +396,14 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_createComplaintVote &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index fb8f2c00..e7614154 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -336,6 +336,28 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, td::Promise promise) { + auto create_writer = [&](std::string path, td::Promise P) { + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", + std::move(path), std::move(data), std::move(P)) + .release(); + }; + add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); +} + +void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_state, + td::Promise promise) { + auto create_writer = [&](std::string path, td::Promise P) { + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", + std::move(path), std::move(write_state), std::move(P)) + .release(); + }; + add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); +} + +void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise, + std::function)> create_writer) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); if (perm_states_.find(hash) != perm_states_.end()) { @@ -353,8 +375,7 @@ void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt master promise.set_value(td::Unit()); } }); - td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", path, std::move(data), std::move(P)) - .release(); + create_writer(std::move(path), std::move(P)); } void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise promise) { diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index dbe6e4d4..31e0e6b6 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -45,6 +45,9 @@ class ArchiveManager : public td::actor::Actor { void add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise); void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, td::Promise promise); + void add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_state, + td::Promise promise); void get_zero_state(BlockIdExt block_id, td::Promise promise); void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, @@ -137,6 +140,8 @@ class ArchiveManager : public td::actor::Actor { PackageId get_max_temp_file_desc_idx(); PackageId get_prev_temp_file_desc_idx(PackageId id); + void add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, + std::function)> create_writer); void written_perm_state(FileReferenceShort id); void persistent_state_gc(FileHash last); diff --git a/validator/db/files-async.hpp b/validator/db/files-async.hpp index 1f2ab5ce..2da534bf 100644 --- a/validator/db/files-async.hpp +++ b/validator/db/files-async.hpp @@ -52,30 +52,50 @@ class WriteFile : public td::actor::Actor { auto res = R.move_as_ok(); auto file = std::move(res.first); auto old_name = res.second; - td::uint64 offset = 0; - while (data_.size() > 0) { - auto R = file.pwrite(data_.as_slice(), offset); - auto s = R.move_as_ok(); - offset += s; - data_.confirm_read(s); + auto status = write_data_(file); + if (!status.is_error()) { + status = file.sync(); + } + if (status.is_error()) { + td::unlink(old_name); + promise_.set_error(std::move(status)); + stop(); + return; } - file.sync().ensure(); if (new_name_.length() > 0) { - td::rename(old_name, new_name_).ensure(); - promise_.set_value(std::move(new_name_)); + status = td::rename(old_name, new_name_); + if (status.is_error()) { + promise_.set_error(std::move(status)); + } else { + promise_.set_value(std::move(new_name_)); + } } else { promise_.set_value(std::move(old_name)); } stop(); } + WriteFile(std::string tmp_dir, std::string new_name, std::function write_data, + td::Promise promise) + : tmp_dir_(tmp_dir), new_name_(new_name), write_data_(std::move(write_data)), promise_(std::move(promise)) { + } WriteFile(std::string tmp_dir, std::string new_name, td::BufferSlice data, td::Promise promise) - : tmp_dir_(tmp_dir), new_name_(new_name), data_(std::move(data)), promise_(std::move(promise)) { + : tmp_dir_(tmp_dir), new_name_(new_name), promise_(std::move(promise)) { + write_data_ = [data_ptr = std::make_shared(std::move(data))] (td::FileFd& fd) { + auto data = std::move(*data_ptr); + td::uint64 offset = 0; + while (data.size() > 0) { + TRY_RESULT(s, fd.pwrite(data.as_slice(), offset)); + offset += s; + data.confirm_read(s); + } + return td::Status::OK(); + }; } private: const std::string tmp_dir_; std::string new_name_; - td::BufferSlice data_; + std::function write_data_; td::Promise promise_; }; diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index ef88ad5f..58e3dd2f 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -276,6 +276,13 @@ void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterc std::move(state), std::move(promise)); } +void RootDb::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state_gen, block_id, masterchain_block_id, + std::move(write_data), std::move(promise)); +} + void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id, diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index c4d642e7..2654d482 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -69,6 +69,9 @@ class RootDb : public Db { void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) override; void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) override; void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/fabric.h b/validator/fabric.h index a10251dd..67b6ae9e 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -46,6 +46,8 @@ td::Result>> create_new_shard_bloc td::Ref create_signature_set(std::vector sig_set); +void run_check_external_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, td::Ref approve_signatures, bool send_broadcast, @@ -81,6 +83,8 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b td::Promise promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); +void run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, td::Ref masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index bc295992..279fb6fb 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -16,6 +16,8 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "auto/tl/ton_api.h" +#include "overlays.h" #include "td/utils/SharedSlice.h" #include "full-node-shard.hpp" #include "full-node-shard-queries.hpp" @@ -79,6 +81,10 @@ void FullNodeShardImpl::create_overlay() { void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { td::actor::send_closure(node_, &FullNodeShardImpl::receive_broadcast, src, std::move(data)); } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); + } Callback(td::actor::ActorId node) : node_(node) { } @@ -95,9 +101,21 @@ void FullNodeShardImpl::create_overlay() { } } +void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + return promise.set_error(B.move_as_error_prefix("failed to parse external message broadcast: ")); + } + + auto q = B.move_as_ok(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_external_message, + std::move(q->message_->data_), std::move(promise)); +} + void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); adnl_id_ = adnl_id; + local_id_ = adnl_id_.pubkey_hash(); create_overlay(); } @@ -804,8 +822,9 @@ void FullNodeShardImpl::sign_new_certificate(PublicKeyHash sign_by) { return; } - ton::overlay::Certificate cert{sign_by, static_cast(td::Clocks::system() + 3600), - overlay::Overlays::max_fec_broadcast_size(), td::BufferSlice{}}; + ton::overlay::Certificate cert{ + sign_by, static_cast(td::Clocks::system() + 3600), overlay::Overlays::max_fec_broadcast_size(), + overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, td::BufferSlice{}}; auto to_sign = cert.to_sign(overlay_id_, local_id_); auto P = td::PromiseCreator::lambda( @@ -830,6 +849,38 @@ void FullNodeShardImpl::signed_new_certificate(ton::overlay::Certificate cert) { td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_); } +void FullNodeShardImpl::sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expire_at, td::uint32 max_size, td::Promise promise) { + auto sign_by = sign_cert_by_; + if (sign_by.is_zero()) { + promise.set_error(td::Status::Error("Node has no key with signing authority")); + return; + } + + ton::overlay::Certificate cert{ + sign_by, static_cast(expire_at), max_size, + overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, td::BufferSlice{}}; + auto to_sign = cert.to_sign(overlay_id_, signed_key); + + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), expire_at = expire_at, max_size = max_size, promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to create certificate: failed to sign: ")); + } else { + auto p = R.move_as_ok(); + auto c = ton::create_serialize_tl_object(p.second.tl(), static_cast(expire_at), max_size, std::move(p.first)); + promise.set_value(std::move(c)); + } + }); + td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_add_get_public_key, sign_by, std::move(to_sign), + std::move(P)); +} + +void FullNodeShardImpl::import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::update_certificate, + adnl_id_, overlay_id_, signed_key, cert); + promise.set_value( td::Unit() ); +} + void FullNodeShardImpl::update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) { if (!client_.empty()) { return; @@ -845,7 +896,7 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ authorized_keys.emplace(key, overlay::Overlays::max_fec_broadcast_size()); } - rules_ = overlay::OverlayPrivacyRules{1 << 14, std::move(authorized_keys)}; + rules_ = overlay::OverlayPrivacyRules{overlay::Overlays::max_fec_broadcast_size(), overlay::CertificateFlags::AllowFec, std::move(authorized_keys)}; td::actor::send_closure(overlays_, &overlay::Overlays::set_privacy_rules, adnl_id_, overlay_id_, rules_); if (update_cert) { @@ -949,8 +1000,7 @@ void FullNodeShardImpl::update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, do } } -void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, - td::BufferSlice data) { +void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data) { auto it = neighbours_.find(adnl_id); if (it == neighbours_.end()) { return; diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 40a422e0..c1712baf 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -42,6 +42,10 @@ class FullNodeShard : public td::actor::Actor { virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast) = 0; + virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; + virtual void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) = 0; + + virtual void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 4aad637b..a2dd5cc4 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -19,6 +19,8 @@ #pragma once #include "full-node-shard.h" +#include "td/actor/PromiseFuture.h" +#include "td/utils/port/Poll.h" namespace ton { @@ -139,6 +141,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); + void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; @@ -167,6 +170,10 @@ class FullNodeShardImpl : public FullNodeShard { void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; + + void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) override; + void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; + void sign_new_certificate(PublicKeyHash sign_by); void signed_new_certificate(ton::overlay::Certificate cert); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 691a8b36..6606f215 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -76,6 +76,27 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr promise.set_value(td::Unit()); } +void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) { + auto it = shards_.find(shard_id); + if(it == shards_.end()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; + } + td::actor::send_closure(it->second, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, std::move(promise)); +} + +void FullNodeImpl::import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) { + auto it = shards_.find(shard_id); + if(it == shards_.end()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + } + td::actor::send_closure(it->second, &FullNodeShard::import_overlay_certificate, signed_key, cert, std::move(promise)); +} + void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { adnl_id_ = adnl_id; @@ -86,6 +107,7 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; virtual void del_permanent_key(PublicKeyHash key, td::Promise promise) = 0; + virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) = 0; + virtual void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) = 0; + virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; static constexpr td::uint32 max_block_size() { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index e9b64f57..6d57f4a8 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -42,6 +42,14 @@ class FullNodeImpl : public FullNode { void add_permanent_key(PublicKeyHash key, td::Promise promise) override; void del_permanent_key(PublicKeyHash key, td::Promise promise) override; + void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) override; + void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) override; + + void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; void add_shard(ShardIdFull shard); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 87aa5494..2dced749 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -103,6 +103,27 @@ class Collator final : public td::actor::Actor { return 2; } + static td::Result> + impl_fetch_config_params(std::unique_ptr config, + Ref* old_mparams, + std::vector* storage_prices, + block::StoragePhaseConfig* storage_phase_cfg, + td::BitArray<256>* rand_seed, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, + WorkchainId wc); + + static td::Result> + impl_create_ordinary_transaction(Ref msg_root, + block::Account* acc, + UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + bool external, LogicalTime after_lt); + private: void start_up() override; void alarm() override; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 9ca4a6cc..5820a2ce 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1556,68 +1556,92 @@ bool Collator::init_lt() { } bool Collator::fetch_config_params() { - old_mparams_ = config_->get_config_param(9); - { - auto res = config_->get_storage_prices(); - if (res.is_error()) { + auto res = impl_fetch_config_params(std::move(config_), + &old_mparams_, &storage_prices_, &storage_phase_cfg_, + &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, + &masterchain_create_fee_, &basechain_create_fee_, + workchain() + ); + if (res.is_error()) { return fatal_error(res.move_as_error()); + } + config_ = res.move_as_ok(); + return true; +} + +td::Result> + Collator::impl_fetch_config_params(std::unique_ptr config, + Ref* old_mparams, + std::vector* storage_prices, + block::StoragePhaseConfig* storage_phase_cfg, + td::BitArray<256>* rand_seed, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, + WorkchainId wc) { + *old_mparams = config->get_config_param(9); + { + auto res = config->get_storage_prices(); + if (res.is_error()) { + return res.move_as_error(); } - storage_prices_ = res.move_as_ok(); + *storage_prices = res.move_as_ok(); } { // generate rand seed - prng::rand_gen().strong_rand_bytes(rand_seed_.data(), 32); - LOG(DEBUG) << "block random seed set to " << rand_seed_.to_hex(); + prng::rand_gen().strong_rand_bytes(rand_seed->data(), 32); + LOG(DEBUG) << "block random seed set to " << rand_seed->to_hex(); } { // compute compute_phase_cfg / storage_phase_cfg - auto cell = config_->get_config_param(is_masterchain() ? 20 : 21); + auto cell = config->get_config_param(wc == ton::masterchainId ? 20 : 21); if (cell.is_null()) { - return fatal_error("cannot fetch current gas prices and limits from masterchain configuration"); + return td::Status::Error(-668, "cannot fetch current gas prices and limits from masterchain configuration"); } - if (!compute_phase_cfg_.parse_GasLimitsPrices(std::move(cell), storage_phase_cfg_.freeze_due_limit, - storage_phase_cfg_.delete_due_limit)) { - return fatal_error("cannot unpack current gas prices and limits from masterchain configuration"); + if (!compute_phase_cfg->parse_GasLimitsPrices(std::move(cell), storage_phase_cfg->freeze_due_limit, + storage_phase_cfg->delete_due_limit)) { + return td::Status::Error(-668, "cannot unpack current gas prices and limits from masterchain configuration"); } - compute_phase_cfg_.block_rand_seed = rand_seed_; - compute_phase_cfg_.libraries = std::make_unique(config_->get_libraries_root(), 256); - compute_phase_cfg_.global_config = config_->get_root_cell(); + compute_phase_cfg->block_rand_seed = *rand_seed; + compute_phase_cfg->libraries = std::make_unique(config->get_libraries_root(), 256); + compute_phase_cfg->global_config = config->get_root_cell(); } { // compute action_phase_cfg block::gen::MsgForwardPrices::Record rec; - auto cell = config_->get_config_param(24); + auto cell = config->get_config_param(24); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return fatal_error("cannot fetch masterchain message transfer prices from masterchain configuration"); + return td::Status::Error(-668, "cannot fetch masterchain message transfer prices from masterchain configuration"); } - action_phase_cfg_.fwd_mc = + action_phase_cfg->fwd_mc = block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - cell = config_->get_config_param(25); + cell = config->get_config_param(25); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return fatal_error("cannot fetch standard message transfer prices from masterchain configuration"); + return td::Status::Error(-668, "cannot fetch standard message transfer prices from masterchain configuration"); } - action_phase_cfg_.fwd_std = + action_phase_cfg->fwd_std = block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - action_phase_cfg_.workchains = &config_->get_workchain_list(); - action_phase_cfg_.bounce_msg_body = (config_->has_capability(ton::capBounceMsgBody) ? 256 : 0); + action_phase_cfg->workchains = &config->get_workchain_list(); + action_phase_cfg->bounce_msg_body = (config->has_capability(ton::capBounceMsgBody) ? 256 : 0); } { // fetch block_grams_created - auto cell = config_->get_config_param(14); + auto cell = config->get_config_param(14); if (cell.is_null()) { - basechain_create_fee_ = masterchain_create_fee_ = td::zero_refint(); + *basechain_create_fee = *masterchain_create_fee = td::zero_refint(); } else { block::gen::BlockCreateFees::Record create_fees; if (!(tlb::unpack_cell(cell, create_fees) && - block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, masterchain_create_fee_) && - block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, basechain_create_fee_))) { - return fatal_error("cannot unpack BlockCreateFees from configuration parameter #14"); + block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, *masterchain_create_fee) && + block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, *basechain_create_fee))) { + return td::Status::Error(-668, "cannot unpack BlockCreateFees from configuration parameter #14"); } } } - return true; + return std::move(config); } bool Collator::compute_minted_amount(block::CurrencyCollection& to_mint) { @@ -1920,7 +1944,6 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr } auto ptr = std::make_unique(workchain(), addr); if (account.is_null()) { - ptr->created = true; if (!ptr->init_new(now_)) { return nullptr; } @@ -2218,75 +2241,25 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { } block::Account* acc = acc_res.move_as_ok(); assert(acc); - if (acc->last_trans_end_lt_ >= start_lt && acc->transactions.empty()) { - fatal_error(PSTRING() << "last transaction time in the state of account " << workchain() << ":" << addr.to_hex() - << " is too large"); - return {}; - } - auto trans_min_lt = start_lt; - if (external) { - // transactions processing external messages must have lt larger than all processed internal messages - trans_min_lt = std::max(trans_min_lt, last_proc_int_msg_.first); - } - std::unique_ptr trans = - std::make_unique(*acc, block::Transaction::tr_ord, trans_min_lt + 1, now_, msg_root); - bool ihr_delivered = false; // FIXME - if (!trans->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { - if (external) { - // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by account " << addr.to_hex() - << " before smart-contract execution"; + + + auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, + &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, + external, last_proc_int_msg_.first + ); + if(res.is_error()) { + auto error = res.move_as_error(); + if(error.code() == -701) { + // ignorable errors + LOG(DEBUG) << error.message(); return {}; } - fatal_error("cannot unpack input message for a new transaction"); - return {}; - } - if (trans->bounce_enabled) { - if (!trans->prepare_storage_phase(storage_phase_cfg_, true)) { - fatal_error("cannot create storage phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!external && !trans->prepare_credit_phase()) { - fatal_error("cannot create credit phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - } else { - if (!external && !trans->prepare_credit_phase()) { - fatal_error("cannot create credit phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!trans->prepare_storage_phase(storage_phase_cfg_, true, true)) { - fatal_error("cannot create storage phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - } - if (!trans->prepare_compute_phase(compute_phase_cfg_)) { - fatal_error("cannot create compute phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!trans->compute_phase->accepted) { - if (external) { - // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by transaction " << addr.to_hex(); - return {}; - } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { - fatal_error("new ordinary transaction for smart contract "s + addr.to_hex() + - " has not been accepted by the smart contract (?)"); - return {}; - } - } - if (trans->compute_phase->success && !trans->prepare_action_phase(action_phase_cfg_)) { - fatal_error("cannot create action phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(action_phase_cfg_)) { - fatal_error("cannot create bounce phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!trans->serialize()) { - fatal_error("cannot serialize new transaction for smart contract "s + addr.to_hex()); + fatal_error(std::move(error)); return {}; } + std::unique_ptr trans = res.move_as_ok(); + if (!trans->update_limits(*block_limit_status_)) { fatal_error("cannot update block limit status to include the new transaction"); return {}; @@ -2296,15 +2269,81 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { fatal_error("cannot commit new transaction for smart contract "s + addr.to_hex()); return {}; } + register_new_msgs(*trans); update_max_lt(acc->last_trans_end_lt_); - // temporary patch to stop producing dangerous block - if (acc->status == block::Account::acc_nonexist) { - block_full_ = true; - } return trans_root; } +// If td::status::error_code == 669 - Fatal Error block can not be produced +// if td::status::error_code == 701 - Transaction can not be included into block, but it's ok (external or too early internal) +td::Result> Collator::impl_create_ordinary_transaction(Ref msg_root, + block::Account* acc, + UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + bool external, LogicalTime after_lt) { + if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { + return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() + << " is too large"); + } + auto trans_min_lt = lt; + if (external) { + // transactions processing external messages must have lt larger than all processed internal messages + trans_min_lt = std::max(trans_min_lt, after_lt); + } + + std::unique_ptr trans = + std::make_unique(*acc, block::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); + bool ihr_delivered = false; // FIXME + if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { + if (external) { + // inbound external message was not accepted + return td::Status::Error(-701,"inbound external message rejected by account "s + acc->addr.to_hex() + + " before smart-contract execution"); + } + return td::Status::Error(-669,"cannot unpack input message for a new transaction"); + } + if (trans->bounce_enabled) { + if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { + return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } else { + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } + if (!trans->prepare_compute_phase(*compute_phase_cfg)) { + return td::Status::Error(-669,"cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->compute_phase->accepted) { + if (external) { + // inbound external message was not accepted + return td::Status::Error(-701,"inbound external message rejected by transaction "s + acc->addr.to_hex()); + } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { + return td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() + + " has not been accepted by the smart contract (?)"); + } + } + if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { + return td::Status::Error(-669,"cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(*action_phase_cfg)) { + return td::Status::Error(-669,"cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->serialize()) { + return td::Status::Error(-669,"cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); + } + return std::move(trans); +} + void Collator::update_max_lt(ton::LogicalTime lt) { CHECK(lt >= start_lt); if (lt > max_lt) { @@ -2451,10 +2490,6 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R if (!insert_out_msg(cb.finalize())) { return -1; } - // 6.5. check for temporary patch can be left here - if (block_full_) { - return 3; - } // 7. check whether the block is full now if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { block_full_ = true; diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 6af721b2..c5f2d097 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -16,19 +16,25 @@ Copyright 2017-2020 Telegram Systems LLP */ + #include "external-message.hpp" +#include "collator-impl.h" #include "vm/boc.h" #include "block/block-parse.h" #include "block/block-auto.h" #include "block/block-db.h" +#include "fabric.h" +#include "td/actor/actor.h" +#include "td/utils/Random.h" +#include "crypto/openssl/rand.hpp" namespace ton { namespace validator { using td::Ref; -ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull addr_prefix) - : root_(std::move(root)), addr_prefix_(addr_prefix), data_(std::move(data)) { +ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull addr_prefix, ton::WorkchainId wc, ton::StdSmcAddress addr) + : root_(std::move(root)), addr_prefix_(addr_prefix), data_(std::move(data)), wc_(wc), addr_(addr) { hash_ = block::compute_file_hash(data_); } @@ -70,7 +76,95 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat if (!dest_prefix.is_valid()) { return td::Status::Error("destination of an inbound external message is an invalid blockchain address"); } - return Ref{true, std::move(data), std::move(ext_msg), dest_prefix}; + ton::StdSmcAddress addr; + ton::WorkchainId wc; + if(!block::tlb::t_MsgAddressInt.extract_std_address(info.dest, wc, addr)) { + return td::Status::Error(PSLICE() << "Can't parse destination address"); + } + + return Ref{true, std::move(data), std::move(ext_msg), dest_prefix, wc, addr}; +} + +void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId manager, + td::Promise promise) { + auto R = create_ext_message(std::move(data)); + if (R.is_error()) { + return promise.set_error(R.move_as_error_prefix("failed to parse external message ")); + } + auto M = R.move_as_ok(); + auto root = M->root_cell(); + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + tlb::unpack_cell_inexact(root, info); // checked in create message + ton::StdSmcAddress addr = M->addr(); + ton::WorkchainId wc = M->wc(); + + run_fetch_account_state(wc, addr, manager, + [promise = std::move(promise), msg_root = root, wc = wc](td::Result,UnixTime,LogicalTime,std::unique_ptr>> res) mutable { + if (res.is_error()) { + promise.set_error(td::Status::Error(PSLICE() << "Failed to get account state")); + } else { + auto tuple = res.move_as_ok(); + block::Account acc; + auto shard_acc = std::move(std::get<0>(tuple)); + auto utime = std::get<1>(tuple); + auto lt = std::get<2>(tuple); + auto config = std::move(std::get<3>(tuple)); + if(!acc.unpack(shard_acc, {}, utime, false)) { + promise.set_error(td::Status::Error(PSLICE() << "Failed to unpack account state")); + } else { + if(run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config))) { + promise.set_value(td::Unit()); + } else { + promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted")); + } + } + } + } + ); +} + +bool ExtMessageQ::run_message_on_account(ton::WorkchainId wc, + block::Account* acc, + UnixTime utime, LogicalTime lt, + td::Ref msg_root, + std::unique_ptr config) { + + Ref old_mparams; + std::vector storage_prices_; + block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; + td::BitArray<256> rand_seed_; + block::ComputePhaseConfig compute_phase_cfg_; + block::ActionPhaseConfig action_phase_cfg_; + td::RefInt256 masterchain_create_fee, basechain_create_fee; + + auto fetch_res = Collator::impl_fetch_config_params(std::move(config), &old_mparams, + &storage_prices_, &storage_phase_cfg_, + &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &masterchain_create_fee, + &basechain_create_fee, wc); + if(fetch_res.is_error()) { + auto error = fetch_res.move_as_error(); + LOG(DEBUG) << "Cannot fetch config params" << error.message(); + return false; + } + + auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, + &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, + true, lt); + if(res.is_error()) { + auto error = res.move_as_error(); + LOG(DEBUG) << "Cannot run message on account" << error.message(); + return false; + } + std::unique_ptr trans = res.move_as_ok(); + + auto trans_root = trans->commit(*acc); + if (trans_root.is_null()) { + LOG(DEBUG) << "cannot commit new transaction for smart contract "; + return false; + } + return true; } } // namespace validator diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index 5bb52b11..230258cd 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -18,9 +18,11 @@ */ #pragma once +#include "interfaces/validator-manager.h" #include "validator/interfaces/external-message.h" #include "auto/tl/ton_api.h" #include "adnl/utils.hpp" +#include "block/transaction.h" namespace ton { @@ -31,6 +33,8 @@ class ExtMessageQ : public ExtMessage { AccountIdPrefixFull addr_prefix_; td::BufferSlice data_; Hash hash_; + ton::WorkchainId wc_; + ton::StdSmcAddress addr_; public: static constexpr unsigned max_ext_msg_size = 65535; @@ -47,8 +51,23 @@ class ExtMessageQ : public ExtMessage { Hash hash() const override { return hash_; } - ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull shard); + ton::WorkchainId wc() const override { + return wc_; + } + + ton::StdSmcAddress addr() const override { + return addr_; + } + + ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull shard, ton::WorkchainId wc, ton::StdSmcAddress addr); static td::Result> create_ext_message(td::BufferSlice data); + static void run_message(td::BufferSlice data, td::actor::ActorId manager, + td::Promise promise); + static bool run_message_on_account(ton::WorkchainId wc, + block::Account* acc, + UnixTime utime, LogicalTime lt, + td::Ref msg_root, + std::unique_ptr config); }; } // namespace validator diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index ac296eb4..196595ae 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -116,6 +116,10 @@ td::Result> create_ext_message(td::BufferSlice data) { return std::move(res); } +void run_check_external_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise) { + ExtMessageQ::run_message(std::move(data), std::move(manager), std::move(promise)); +} + td::Result> create_ihr_message(td::BufferSlice data) { TRY_RESULT(res, IhrMessageQ::create_ihr_message(std::move(data))); return std::move(res); @@ -237,6 +241,11 @@ void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) { + LiteQuery::fetch_account_state(wc, addr, std::move(manager), std::move(promise)); +} + void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, td::Ref masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index b3c0f3eb..90d974e0 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -58,15 +58,28 @@ void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId("litequery", std::move(data), std::move(manager), std::move(promise)).release(); } +void LiteQuery::fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) { + td::actor::create_actor("litequery", wc, acc_addr, std::move(manager), std::move(promise)).release(); +} + LiteQuery::LiteQuery(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise) : query_(std::move(data)), manager_(std::move(manager)), promise_(std::move(promise)) { timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); } +LiteQuery::LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) + : manager_(std::move(manager)), acc_state_promise_(std::move(promise)), acc_workchain_(wc), acc_addr_(acc_addr) { + timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); +} + void LiteQuery::abort_query(td::Status reason) { LOG(INFO) << "aborted liteserver query: " << reason.to_string(); - if (promise_) { + if (acc_state_promise_) { + acc_state_promise_.set_error(std::move(reason)); + } else if (promise_) { promise_.set_error(std::move(reason)); } stop(); @@ -111,6 +124,11 @@ bool LiteQuery::finish_query(td::BufferSlice result) { void LiteQuery::start_up() { alarm_timestamp() = timeout_; + if(acc_state_promise_) { + td::actor::send_closure_later(actor_id(this),&LiteQuery::perform_fetchAccountState); + return; + } + auto F = fetch_tl_object(std::move(query_), true); if (F.is_error()) { abort_query(F.move_as_error()); @@ -180,6 +198,9 @@ void LiteQuery::start_up() { this->perform_runSmcMethod(ton::create_block_id(q.id_), static_cast(q.account_->workchain_), q.account_->id_, q.mode_, q.method_id_, std::move(q.params_)); }, + [&](lite_api::liteServer_getLibraries& q) { + this->perform_getLibraries(q.library_list_); + }, [&](auto& obj) { this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); })); } @@ -205,17 +226,23 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { } td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [Self = actor_id(this), mode](td::Result, BlockIdExt>> res) { + [Self = actor_id(this), return_state = bool(acc_state_promise_), mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto pair = res.move_as_ok(); - td::actor::send_closure_later(Self, &LiteQuery::continue_getMasterchainInfo, std::move(pair.first), + auto func = return_state ? &LiteQuery::gotMasterchainInfoForAccountState : &LiteQuery::continue_getMasterchainInfo; + td::actor::send_closure_later(Self, func, std::move(pair.first), pair.second, mode); } }); } +void LiteQuery::gotMasterchainInfoForAccountState(Ref mc_state, BlockIdExt blkid, + int mode) { + perform_getAccountState(blkid, acc_workchain_, acc_addr_, 0x80000000); +} + void LiteQuery::continue_getMasterchainInfo(Ref mc_state, BlockIdExt blkid, int mode) { LOG(INFO) << "obtained data for getMasterchainInfo() : last block = " << blkid.to_str(); @@ -431,15 +458,25 @@ void LiteQuery::continue_getZeroState(BlockIdExt blkid, td::BufferSlice state) { void LiteQuery::perform_sendMessage(td::BufferSlice data) { LOG(INFO) << "started a sendMessage(<" << data.size() << " bytes>) liteserver query"; - auto res = ton::validator::create_ext_message(std::move(data)); - if (res.is_error()) { - abort_query(res.move_as_error()); - return; - } - LOG(INFO) << "sending an external message to validator manager"; - td::actor::send_closure_later(manager_, &ValidatorManager::send_external_message, res.move_as_ok()); - auto b = ton::create_serialize_tl_object(1); - finish_query(std::move(b)); + td::actor::send_closure_later( + manager_, &ValidatorManager::check_external_message, data.clone(), + [Self = actor_id(this), data = std::move(data), manager = manager_](td::Result res) { + if(res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot apply external message to current state : "s)); + } else { + auto crm = ton::validator::create_ext_message(data.clone()); + if (crm.is_error()) { + //UNREACHABLE, checks in check_external_message, + td::actor::send_closure(Self, &LiteQuery::abort_query, + crm.move_as_error()); + } + LOG(INFO) << "sending an external message to validator manager"; + td::actor::send_closure_later(manager, &ValidatorManager::send_external_message, crm.move_as_ok()); + auto b = ton::create_serialize_tl_object(1); + td::actor::send_closure(Self, &LiteQuery::finish_query, std::move(b)); + } + }); } bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { @@ -702,6 +739,10 @@ void LiteQuery::continue_getAccountState_0(Ref request_mc_block_data(blkid); } +void LiteQuery::perform_fetchAccountState() { + perform_getMasterchainInfo(-1); +} + void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode, td::int64 method_id, td::BufferSlice params) { LOG(INFO) << "started a runSmcMethod(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ", " @@ -743,6 +784,73 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St perform_getAccountState(blkid, workchain, addr, mode | 0x10000); } +void LiteQuery::perform_getLibraries(std::vector library_list) { + LOG(INFO) << "started a getLibraries() liteserver query"; + if (library_list.size() > 16) { + LOG(INFO) << "too many libraries requested, returning only first 16"; + library_list.resize(16); + } + sort( library_list.begin(), library_list.end() ); + library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() ); + td::actor::send_closure_later( + manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, + [Self = actor_id(this), library_list](td::Result, BlockIdExt>> res) -> void { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + auto pair = res.move_as_ok(); + td::actor::send_closure_later(Self, &LiteQuery::continue_getLibraries, std::move(pair.first), + pair.second, library_list); + } + }); +} + +void LiteQuery::continue_getLibraries(Ref mc_state, BlockIdExt blkid, std::vector library_list) { + LOG(INFO) << "obtained last masterchain block = " << blkid.to_str(); + base_blk_id_ = blkid; + CHECK(mc_state.not_null()); + mc_state_ = Ref(std::move(mc_state)); + CHECK(mc_state_.not_null()); + + auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries); + if (rconfig.is_error()) { + fatal_error("cannot extract library list block configuration from masterchain state"); + return; + } + auto config = rconfig.move_as_ok(); + + if (false) { + std::ostringstream os; + vm::load_cell_slice(config->get_libraries_root()).print_rec(os); + LOG(INFO) << "\n" << os.str(); + + auto lib_dict = std::make_unique(config->get_libraries_root(), 256); + for (auto k: *lib_dict) { + std::ostringstream oss; + k.second->print_rec(oss); + LOG(INFO) << "library " << k.first.to_hex(256) << ": \n" << oss.str(); + } + } + + std::vector> a; + for (const auto& hash : library_list) { + LOG(INFO) << "looking for library " << hash.to_hex(); + auto libres = config->lookup_library(hash); + if (libres.is_null()) { + LOG(INFO) << "library lookup result is null"; + continue; + } + auto data = vm::std_boc_serialize(libres); + if (data.is_error()) { + LOG(WARNING) << "library serialization failed: " << data.move_as_error().to_string(); + continue; + } + a.push_back(ton::create_tl_object(hash, data.move_as_ok())); + } + auto b = ton::create_serialize_tl_object(std::move(a)); + finish_query(std::move(b)); +} + void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt) { LOG(INFO) << "started a getOneTransaction(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << "," << lt << ") liteserver query"; @@ -1010,6 +1118,19 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { } vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, block::tlb::aug_ShardAccounts}; auto acc_csr = accounts_dict.lookup(acc_addr_); + if (mode_ & 0x80000000) { + auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), 0xFFFF); + if (config.is_error()) { + fatal_error(config.move_as_error()); + return; + } + auto rconfig = config.move_as_ok(); + acc_state_promise_.set_value(std::make_tuple( + std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig) + )); + return; + } + Ref acc_root; if (acc_csr.not_null()) { acc_root = acc_csr->prefetch_ref(); @@ -1099,7 +1220,7 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice long long gas_limit = client_method_gas_limit; LOG(DEBUG) << "creating VM with gas limit " << gas_limit; // **** INIT VM **** - vm::GasLimits gas{gas_limit}; + vm::GasLimits gas{gas_limit, gas_limit}; vm::VmState vm{std::move(code), std::move(stack_), gas, 1, std::move(data), vm::VmLog::Null()}; auto c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref(acc.addr->clone()), balance); vm.set_c7(c7); // tuple with SmartContractInfo diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index f6353c41..f1918ceb 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -26,6 +26,8 @@ #include "block.hpp" #include "shard.hpp" #include "proof.hpp" +#include "block/block-auto.h" + namespace ton { @@ -37,6 +39,9 @@ class LiteQuery : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise promise_; + + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> acc_state_promise_; + int pending_{0}; int mode_{0}; WorkchainId acc_workchain_; @@ -71,9 +76,14 @@ class LiteQuery : public td::actor::Actor { }; // version 1.1; +1 = build block proof chains, +2 = masterchainInfoExt, +4 = runSmcMethod LiteQuery(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); static void run_query(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + static void fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); + private: bool fatal_error(td::Status error); bool fatal_error(std::string err_msg, int err_code = -400); @@ -87,6 +97,7 @@ class LiteQuery : public td::actor::Actor { void perform_getVersion(); void perform_getMasterchainInfo(int mode); void continue_getMasterchainInfo(Ref mc_state, BlockIdExt blkid, int mode); + void gotMasterchainInfoForAccountState(Ref mc_state, BlockIdExt blkid, int mode); void perform_getBlock(BlockIdExt blkid); void continue_getBlock(BlockIdExt blkid, Ref block); void perform_getBlockHeader(BlockIdExt blkid, int mode); @@ -99,10 +110,13 @@ class LiteQuery : public td::actor::Actor { void continue_getAccountState_0(Ref mc_state, BlockIdExt blkid); void continue_getAccountState(); void finish_getAccountState(td::BufferSlice shard_proof); + void perform_fetchAccountState(); void perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode, td::int64 method_id, td::BufferSlice params); void finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice state_proof, Ref acc_root, UnixTime gen_utime, LogicalTime gen_lt); + void perform_getLibraries(std::vector library_list); + void continue_getLibraries(Ref mc_state, BlockIdExt blkid, std::vector library_list); void perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt); void continue_getOneTransaction(); void perform_getTransactions(WorkchainId workchain, StdSmcAddress addr, LogicalTime lt, Bits256 hash, unsigned count); diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 18f71d32..a96f1a81 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -26,6 +26,7 @@ #include "vm/cells/MerkleUpdate.h" #include "block/block-parse.h" #include "block/block-auto.h" +#include "td/utils/filesystem.h" #define LAZY_STATE_DESERIALIZE 1 @@ -301,6 +302,30 @@ td::Result ShardStateQ::serialize() const { return st_res.move_as_ok(); } +td::Status ShardStateQ::serialize_to_file(td::FileFd& fd) const { + td::PerfWarningTimer perf_timer_{"serializestate", 0.1}; + if (!data.is_null()) { + auto cur_data = data.clone(); + while (cur_data.size() > 0) { + TRY_RESULT(s, fd.write(cur_data.as_slice())); + cur_data.confirm_read(s); + } + return td::Status::OK(); + } + if (root.is_null()) { + return td::Status::Error(-666, "cannot serialize an uninitialized state"); + } + vm::BagOfCells new_boc; + new_boc.set_root(root); + TRY_STATUS(new_boc.import_cells()); + auto st_res = new_boc.serialize_to_file(fd, 31); + if (st_res.is_error()) { + LOG(ERROR) << "cannot serialize a shardchain state"; + return st_res.move_as_error(); + } + return td::Status::OK(); +} + MasterchainStateQ::MasterchainStateQ(const BlockIdExt& _id, td::BufferSlice _data) : MasterchainState(), ShardStateQ(_id, std::move(_data)) { } diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 862b0fc5..0056df11 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -87,6 +87,7 @@ class ShardStateQ : virtual public ShardState { td::Result> merge_with(const ShardState& with) const override; td::Result, Ref>> split() const override; td::Result serialize() const override; + td::Status serialize_to_file(td::FileFd& fd) const override; }; #if TD_MSVC diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index fc90be1d..02a16f83 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -4073,7 +4073,6 @@ std::unique_ptr ValidateQuery::make_account_from(td::ConstBitPtr Ref extra) { auto ptr = std::make_unique(workchain(), addr); if (account.is_null()) { - ptr->created = true; if (!ptr->init_new(now_)) { return nullptr; } @@ -4308,7 +4307,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } } if (is_first && is_masterchain() && account.is_special && account.tick && - (tag != block::gen::TransactionDescr::trans_tick_tock || (td_cs.prefetch_ulong(4) & 1)) && !account.created) { + (tag != block::gen::TransactionDescr::trans_tick_tock || (td_cs.prefetch_ulong(4) & 1)) && account.orig_status == block::Account::acc_active) { return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex() << " is the first transaction for this special tick account in this block, but the " "transaction is not a tick transaction"); diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index f49a7819..9983572b 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -53,6 +53,9 @@ class Db : public td::actor::Actor { virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) = 0; + virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) = 0; virtual void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/interfaces/external-message.h b/validator/interfaces/external-message.h index f6d9d600..871c624a 100644 --- a/validator/interfaces/external-message.h +++ b/validator/interfaces/external-message.h @@ -35,6 +35,8 @@ class ExtMessage : public td::CntObject { virtual td::BufferSlice serialize() const = 0; virtual td::Ref root_cell() const = 0; virtual Hash hash() const = 0; + virtual ton::WorkchainId wc() const = 0; + virtual ton::StdSmcAddress addr() const = 0; }; } // namespace validator diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 0cfbda43..96ca8da2 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -55,6 +55,7 @@ class ShardState : public td::CntObject { virtual td::Result, td::Ref>> split() const = 0; virtual td::Result serialize() const = 0; + virtual td::Status serialize_to_file(td::FileFd& fd) const = 0; }; class MasterchainState : virtual public ShardState { diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 1d4a7c9f..42e99ecc 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -57,6 +57,9 @@ class ValidatorManager : public ValidatorManagerInterface { td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) = 0; + virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index e43e37b2..404f7fe8 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -680,6 +680,13 @@ void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, Bloc std::move(promise)); } +void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, + std::move(write_data), std::move(promise)); +} + void ValidatorManagerImpl::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) { td::actor::send_closure(db_, &Db::store_zero_state_file, block_id, std::move(state), std::move(promise)); diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 71818e6e..bb740b4c 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -124,6 +124,9 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void check_external_message(td::BufferSlice data, td::Promise promise) override { + UNREACHABLE(); + } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; @@ -140,6 +143,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 506ee036..71157dcc 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -144,6 +144,9 @@ class ValidatorManagerImpl : public ValidatorManager { void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void check_external_message(td::BufferSlice data, td::Promise promise) override { + UNREACHABLE(); + } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { UNREACHABLE(); @@ -166,6 +169,11 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override { UNREACHABLE(); } + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) override { + UNREACHABLE(); + } void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override { UNREACHABLE(); } diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index cca320ff..1a322c02 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -227,7 +227,7 @@ void ValidatorManagerMasterchainReiniter::choose_masterchain_state() { } if (!p || ValidatorManager::is_persistent_state(h->unix_time(), p->unix_time())) { auto ttl = ValidatorManager::persistent_state_ttl(h->unix_time()); - double time_to_download = 3600; + double time_to_download = 3600 * 3; if (ttl > td::Clocks::system() + time_to_download) { handle = h; break; @@ -259,7 +259,7 @@ void ValidatorManagerMasterchainReiniter::download_masterchain_state() { } }); td::actor::create_actor("downloadstate", block_id_, block_id_, 2, manager_, - td::Timestamp::in(3600), std::move(P)) + td::Timestamp::in(3600 * 3), std::move(P)) .release(); } @@ -267,7 +267,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Ref{std::move(state)}; CHECK(handle_->received_state()); CHECK(handle_->is_applied()); - + LOG(INFO) << "downloaded masterchain state"; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_all_shards); @@ -276,6 +276,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Ref R) { R.ensure(); @@ -286,6 +287,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_all_shards() { void ValidatorManagerMasterchainReiniter::finish() { CHECK(handle_->id().id.seqno == 0 || handle_->is_key_block()); promise_.set_value(ValidatorManagerInitResult{handle_, state_, std::move(client_), handle_, state_, handle_}); + LOG(INFO) << "persistent state download finished"; stop(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3abea6e8..28e5cd3a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -366,18 +366,33 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { if (!is_validator()) { return; } + if( ext_messages_.size() > max_mempool_num() ) { + return; + } auto R = create_ext_message(std::move(data)); if (R.is_error()) { VLOG(VALIDATOR_NOTICE) << "dropping bad ihr message: " << R.move_as_error(); return; } - auto M = std::make_unique>(R.move_as_ok()); - auto id = M->ext_id(); - if (ext_messages_hashes_.count(id.hash) == 0) { - ext_messages_.emplace(id, std::move(M)); - ext_messages_hashes_.emplace(id.hash, id); + add_external_message(R.move_as_ok()); +} + +void ValidatorManagerImpl::add_external_message(td::Ref msg) { + auto message = std::make_unique>(msg); + auto id = message->ext_id(); + auto address = message->address(); + unsigned long per_address_limit = 256; + if(ext_addr_messages_.count(address) < per_address_limit) { + if (ext_messages_hashes_.count(id.hash) == 0) { + ext_messages_.emplace(id, std::move(message)); + ext_messages_hashes_.emplace(id.hash, id); + ext_addr_messages_[address].emplace(id.hash, id); + } } } +void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise promise) { + run_check_external_message(std::move(data), actor_id(this), std::move(promise)); +} void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { if (!is_validator()) { @@ -756,6 +771,7 @@ void ValidatorManagerImpl::get_external_messages(ShardIdFull shard, break; } if (it->second->expired()) { + ext_addr_messages_[it->second->address()].erase(it->first.hash); ext_messages_hashes_.erase(it->first.hash); it = ext_messages_.erase(it); continue; @@ -804,17 +820,20 @@ void ValidatorManagerImpl::complete_external_messages(std::vectorsecond]->address()].erase(it->first); CHECK(ext_messages_.erase(it->second)); ext_messages_hashes_.erase(it); } } + unsigned long soft_mempool_limit = 1024; for (auto &hash : to_delay) { auto it = ext_messages_hashes_.find(hash); if (it != ext_messages_hashes_.end()) { auto it2 = ext_messages_.find(it->second); - if (it2->second->can_postpone()) { + if ((ext_messages_.size() < soft_mempool_limit) && it2->second->can_postpone()) { it2->second->postpone(); } else { + ext_addr_messages_[it2->second->address()].erase(it2->first.hash); ext_messages_.erase(it2); ext_messages_hashes_.erase(it); } @@ -1032,6 +1051,13 @@ void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, Bloc std::move(promise)); } +void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, + std::move(write_data), std::move(promise)); +} + void ValidatorManagerImpl::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) { td::actor::send_closure(db_, &Db::store_zero_state_file, block_id, std::move(state), std::move(promise)); @@ -1328,7 +1354,7 @@ void ValidatorManagerImpl::send_get_zero_state_request(BlockIdExt id, td::uint32 void ValidatorManagerImpl::send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, td::Promise promise) { - callback_->download_persistent_state(id, masterchain_block_id, priority, td::Timestamp::in(3600.0), + callback_->download_persistent_state(id, masterchain_block_id, priority, td::Timestamp::in(3600 * 3), std::move(promise)); } @@ -1349,6 +1375,7 @@ void ValidatorManagerImpl::send_get_next_key_blocks_request(BlockIdExt block_id, void ValidatorManagerImpl::send_external_message(td::Ref message) { callback_->send_ext_message(message->shard(), message->serialize()); + add_external_message(std::move(message)); } void ValidatorManagerImpl::send_ihr_message(td::Ref message) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 672085d0..123625b2 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -20,6 +20,8 @@ #include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "td/actor/PromiseFuture.h" +#include "td/utils/port/Poll.h" #include "validator-group.hpp" #include "shard-client.hpp" #include "manager-init.h" @@ -72,6 +74,9 @@ class MessageExt { auto hash() const { return message_->hash(); } + auto address() const { + return std::make_pair(message_->wc(), message_->addr()); + } bool is_active() { if (!active_) { if (reactivate_at_.is_in_past()) { @@ -208,6 +213,7 @@ class ValidatorManagerImpl : public ValidatorManager { // DATA FOR COLLATOR std::map> shard_blocks_; std::map, std::unique_ptr>> ext_messages_; + std::map, std::map>> ext_addr_messages_; std::map> ext_messages_hashes_; // IHR ? std::map, std::unique_ptr>> ihr_messages_; @@ -325,6 +331,9 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void add_external_message(td::Ref message); + void check_external_message(td::BufferSlice data, td::Promise promise) override; + void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; @@ -338,6 +347,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -575,6 +587,9 @@ class ValidatorManagerImpl : public ValidatorManager { double block_ttl() const { return opts_->block_ttl(); } + double max_mempool_num() const { + return opts_->max_mempool_num(); + } private: std::map> shard_client_waiters_; diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 0c71133b..fedfaae8 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -53,7 +53,7 @@ DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_i void DownloadState::abort_query(td::Status reason) { if (promise_) { if (reason.code() == ErrorCode::notready || reason.code() == ErrorCode::timeout) { - VLOG(FULL_NODE_DEBUG) << "failed to download state " << block_id_ << "from " << download_from_ << ": " << reason; + VLOG(FULL_NODE_DEBUG) << "failed to download state " << block_id_ << " from " << download_from_ << ": " << reason; } else { VLOG(FULL_NODE_NOTICE) << "failed to download state " << block_id_ << " from " << download_from_ << ": " << reason; @@ -115,6 +115,7 @@ void DownloadState::got_block_handle(BlockHandle handle) { void DownloadState::got_node_to_download(adnl::AdnlNodeIdShort node) { download_from_ = node; + LOG(INFO) << "downloading state " << block_id_ << " from " << download_from_; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { if (R.is_error()) { @@ -148,6 +149,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { abort_query(F.move_as_error()); return; } + prev_logged_timer_ = td::Timer(); ton_api::downcast_call( *F.move_as_ok().get(), @@ -187,6 +189,14 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques sum_ += data.size(); parts_.push_back(std::move(data)); + double elapsed = prev_logged_timer_.elapsed(); + if (elapsed > 10.0) { + prev_logged_timer_ = td::Timer(); + LOG(INFO) << "downloading state " << block_id_ << ": total=" << sum_ << + " (" << double(sum_ - prev_logged_sum_) / elapsed << " B/s)"; + prev_logged_sum_ = sum_; + } + if (last_part) { td::BufferSlice res{td::narrow_cast(sum_)}; auto S = res.as_slice(); @@ -224,6 +234,7 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques void DownloadState::got_block_state(td::BufferSlice data) { state_ = std::move(data); + LOG(INFO) << "finished downloading state " << block_id_ << ": total=" << sum_; finish_query(); } diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index d965e3c7..a586f61f 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -72,6 +72,9 @@ class DownloadState : public td::actor::Actor { td::BufferSlice state_; std::vector parts_; td::uint64 sum_ = 0; + + td::uint64 prev_logged_sum_ = 0; + td::Timer prev_logged_timer_; }; } // namespace fullnode diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 5cafd3ca..c7d87fb4 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -98,8 +98,8 @@ void ShardClient::start_up_init_mode() { }); td::actor::create_actor("downloadstate", shard->top_block_id(), - masterchain_block_handle_->id(), 2, manager_, td::Timestamp::in(3600), - std::move(P)) + masterchain_block_handle_->id(), 2, manager_, + td::Timestamp::in(3600 * 3), std::move(P)) .release(); } } diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 802159c0..82cf31cc 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -81,6 +81,27 @@ void AsyncStateSerializer::alarm() { td::actor::send_closure(manager_, &ValidatorManager::get_top_masterchain_block, std::move(P)); } +void AsyncStateSerializer::request_masterchain_state() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, + R.move_as_error_prefix("failed to get masterchain state: ")); + } else { + td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state, + td::Ref(R.move_as_ok())); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); +} + +void AsyncStateSerializer::request_shard_state(BlockIdExt shard) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok()); + }); + return td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shard, true, std::move(P)); +} + void AsyncStateSerializer::next_iteration() { if (running_) { return; @@ -101,33 +122,28 @@ void AsyncStateSerializer::next_iteration() { CHECK(masterchain_handle_->id() == last_block_id_); if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { - if (masterchain_state_.is_null()) { + if (!have_masterchain_state_) { + LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().seqno(); + // block next attempts immediately, but send actual request later running_ = true; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, - R.move_as_error_prefix("failed to get masterchain state: ")); - } else { - td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state, - td::Ref(R.move_as_ok())); - } - }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); + delay_action( + [SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_masterchain_state); }, + td::Timestamp::in(td::Random::fast(0, 3600))); return; } while (next_idx_ < shards_.size()) { if (!need_monitor(shards_[next_idx_].shard_full())) { next_idx_++; } else { - running_ = true; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok()); - }); - td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shards_[next_idx_], true, std::move(P)); + // block next attempts immediately, but send actual request later + running_ = true; + delay_action( + [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, + td::Timestamp::in(td::Random::fast(0, 4 * 3600))); return; } } + LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().seqno(); last_key_block_ts_ = masterchain_handle_->unix_time(); last_key_block_id_ = masterchain_handle_->id(); } @@ -144,7 +160,7 @@ void AsyncStateSerializer::next_iteration() { } if (masterchain_handle_->inited_next_left()) { last_block_id_ = masterchain_handle_->one_next(true); - masterchain_state_ = td::Ref{}; + have_masterchain_state_ = false; masterchain_handle_ = nullptr; saved_to_db_ = false; shards_.clear(); @@ -168,28 +184,30 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { } void AsyncStateSerializer::got_masterchain_state(td::Ref state) { - masterchain_state_ = state; + LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().seqno(); + have_masterchain_state_ = true; CHECK(next_idx_ == 0); CHECK(shards_.size() == 0); - auto vec = masterchain_state_->get_shards(); - shards_.push_back(masterchain_handle_->id()); + auto vec = state->get_shards(); for (auto &v : vec) { shards_.push_back(v->top_block_id()); } - auto B = masterchain_state_->serialize(); - B.ensure(); + auto write_data = [state] (td::FileFd& fd) { + return state->serialize_to_file(fd); + }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &AsyncStateSerializer::stored_masterchain_state); }); - td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, masterchain_handle_->id(), - masterchain_handle_->id(), B.move_as_ok(), std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, masterchain_handle_->id(), + masterchain_handle_->id(), write_data, std::move(P)); } void AsyncStateSerializer::stored_masterchain_state() { + LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().seqno(); running_ = false; next_iteration(); } @@ -207,14 +225,17 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { } void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state) { - auto B = state->serialize().move_as_ok(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + LOG(INFO) << "serializing shard state " << handle->id().seqno(); + auto write_data = [state] (td::FileFd& fd) { + return state->serialize_to_file(fd); + }; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { R.ensure(); + LOG(INFO) << "finished serializing shard state " << handle->id().seqno(); td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); - td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, handle->id(), - masterchain_handle_->id(), std::move(B), std::move(P)); - LOG(INFO) << "storing persistent state for " << masterchain_handle_->id().seqno() << ":" << handle->id().id.shard; + td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), + masterchain_handle_->id(), write_data, std::move(P)); next_idx_++; } diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 3b8d7052..14261df7 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -43,7 +43,7 @@ class AsyncStateSerializer : public td::actor::Actor { td::uint32 next_idx_ = 0; BlockHandle masterchain_handle_; - td::Ref masterchain_state_; + bool have_masterchain_state_ = false; std::vector shards_; @@ -65,6 +65,9 @@ class AsyncStateSerializer : public td::actor::Actor { void got_self_state(AsyncSerializerState state); void got_init_handle(BlockHandle handle); + void request_masterchain_state(); + void request_shard_state(BlockIdExt shard); + void next_iteration(); void got_top_masterchain_handle(BlockIdExt block_id); void got_masterchain_handle(BlockHandle handle_); diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 30bb050b..93fe05e6 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -27,10 +27,11 @@ namespace validator { td::Ref ValidatorManagerOptions::create( BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, + double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, + max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); } diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 92597156..e794166f 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -50,6 +50,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double state_ttl() const override { return state_ttl_; } + double max_mempool_num() const override { + return max_mempool_num_; + } double archive_ttl() const override { return archive_ttl_; } @@ -130,6 +133,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_state_ttl(double value) override { state_ttl_ = value; } + void set_max_mempool_num(double value) override { + max_mempool_num_ = value; + } void set_archive_ttl(double value) override { archive_ttl_ = value; } @@ -163,7 +169,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, double sync_blocks_before, - double block_ttl, double state_ttl, + double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) @@ -173,6 +179,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { , sync_blocks_before_(sync_blocks_before) , block_ttl_(block_ttl) , state_ttl_(state_ttl) + , max_mempool_num_(max_mempool_num) , archive_ttl_(archive_ttl) , key_proof_ttl_(key_proof_ttl) , initial_sync_disabled_(initial_sync_disabled) { @@ -186,6 +193,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double sync_blocks_before_; double block_ttl_; double state_ttl_; + double max_mempool_num_; double archive_ttl_; double key_proof_ttl_; bool initial_sync_disabled_; diff --git a/validator/validator.h b/validator/validator.h index 54c35ee7..46110c92 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -56,6 +56,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual double sync_blocks_before() const = 0; virtual double block_ttl() const = 0; virtual double state_ttl() const = 0; + virtual double max_mempool_num() const = 0; virtual double archive_ttl() const = 0; virtual double key_proof_ttl() const = 0; virtual bool initial_sync_disabled() const = 0; @@ -81,6 +82,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; virtual void set_state_ttl(double value) = 0; + virtual void set_max_mempool_num(double value) = 0; virtual void set_archive_ttl(double value) = 0; virtual void set_key_proof_ttl(double value) = 0; virtual void set_initial_sync_disabled(bool value) = 0; @@ -94,9 +96,9 @@ struct ValidatorManagerOptions : public td::CntObject { BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard = [](ShardIdFull, CatchainSeqno, ShardCheckMode) { return true; }, - bool allow_blockchain_init = false, double sync_blocks_before = 300, - double block_ttl = 86400 * 7, double state_ttl = 3600, - double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, + bool allow_blockchain_init = false, double sync_blocks_before = 86400, double block_ttl = 86400 * 7, + double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, + double max_mempool_num = 999999, bool initial_sync_disabled = false); }; @@ -176,6 +178,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void write_handle(BlockHandle handle, td::Promise promise) = 0; virtual void new_external_message(td::BufferSlice data) = 0; + virtual void check_external_message(td::BufferSlice data, td::Promise promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0;