Merge pull request #371 from ton-blockchain/release-candidate

Merge updates

CMakeLists.txt

@@ -423,6 +423,9 @@ target_link_libraries(test-vm PRIVATE ton_crypto fift-lib)
 add_executable(test-smartcont test/test-td-main.cpp ${SMARTCONT_TEST_SOURCE})
 target_link_libraries(test-smartcont PRIVATE smc-envelope fift-lib ton_db)
 
+add_executable(test-bigint ${BIGINT_TEST_SOURCE})
+target_link_libraries(test-bigint PRIVATE ton_crypto)
+
 add_executable(test-cells test/test-td-main.cpp ${CELLS_TEST_SOURCE})
 target_link_libraries(test-cells PRIVATE ton_crypto)
 
@@ -523,6 +526,7 @@ if (HAS_PARENT)
           ${FEC_TEST_SOURCE}
           ${ED25519_TEST_SOURCE}
           ${TONDB_TEST_SOURCE}
+          ${BIGNUM_TEST_SOURCE}
           ${CELLS_TEST_SOURCE} # ${TONVM_TEST_SOURCE} ${FIFT_TEST_SOURCE} ${TONLIB_ONLINE_TEST_SOURCE}
           PARENT_SCOPE)
 endif()
@@ -536,6 +540,7 @@ set(TEST_OPTIONS "--regression ${CMAKE_CURRENT_SOURCE_DIR}/test/regression-tests
 separate_arguments(TEST_OPTIONS)
 add_test(test-ed25519-crypto crypto/test-ed25519-crypto)
 add_test(test-ed25519 test-ed25519)
+add_test(test-bigint test-bigint)
 add_test(test-vm test-vm ${TEST_OPTIONS})
 add_test(test-fift test-fift ${TEST_OPTIONS})
 add_test(test-cells test-cells ${TEST_OPTIONS})

Changelog.md

@@ -0,0 +1,14 @@
+## 05.2022 Update
+* Initial synchronization improved: adjusted timeouts for state download and the way of choosing which state to download. Nodes with low network speed and/or bad connectivity will synchronize faster and consistently.
+* Improved peer-to-peer network stability and DDoS resistance: now peers will only relay valid messages to the network. Large messages, which require splitting for relaying, will be retranslated as well, but only after the node gets all parts, and reassembles and checks them. Validators may sign certificates for network peers, which allow relaying large messages by parts without checks. It is used now by validators to faster relay new blocks. Sign and import certificate commands are exposed via `validator-engine-console`.
+* Fixed some rare edge cases in TVM arithmetic operations related to big numbers (`2**63+`)
+* Improved fixes used to combat wrong activate-destruct-activate contract behavior last November.
+* Improved tonlib: support libraries (with client-side caching), getmethods completely fill c7 register, getmethods support slice arguments, improved messages listing for transactions, added extended block header params, added getConfig method.
+* RocksDB updated to a newer version.
+* Improved persistent state serialization: memory usage during serialization was optimized; the start of serialization on different nodes was sparsed.
+* FunC update: support for string literals and constants (including precompiled constant expressions), semver, `include` expressions.
+* Fixed rarely manifested bugs in `Asm.fif`.
+* LiteClient supports key as cli parameter.
+* Improved Liteserver DoS resistance for running getmethods.
+
+Besides the work of the core team, this update is based on the efforts of @tvorogme (added support for slice arguments and noted bugs in Asm.fif), @akifoq (fixed bug in Asm.fif), @cryshado (noted strange behavior of LS, which, upon inspection, turned out to be a vector of DoS attack).

catchain/catchain-receiver.cpp

@@ -486,7 +486,7 @@ void CatChainReceiverImpl::start_up() {
   }
   td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay,
                           get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids),
-                          make_callback(), overlay::OverlayPrivacyRules{0, std::move(root_keys)});
+                          make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)});
 
   CHECK(root_block_);
 

crypto/CMakeLists.txt

@@ -259,6 +259,11 @@ set(FIFT_TEST_SOURCE
   PARENT_SCOPE
 )
 
+set(BIGINT_TEST_SOURCE
+  ${CMAKE_CURRENT_SOURCE_DIR}/test/test-bigint.cpp
+  PARENT_SCOPE
+)
+
 
 add_library(ton_crypto STATIC ${TON_CRYPTO_SOURCE})
 target_include_directories(ton_crypto PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
@@ -293,9 +298,14 @@ add_library(src_parser ${PARSER_SOURCE})
 target_include_directories(src_parser PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
 target_link_libraries(src_parser PUBLIC ton_crypto)
 
+add_library(ton_block ${BLOCK_SOURCE})
+target_include_directories(ton_block PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/block> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
+target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api)
+
 add_executable(func func/func.cpp ${FUNC_LIB_SOURCE})
 target_include_directories(func PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
-target_link_libraries(func PUBLIC ton_crypto src_parser git)
+target_link_libraries(func PUBLIC ton_crypto src_parser git ton_block)
 if (WINGETOPT_FOUND)
   target_link_libraries_system(func wingetopt)
 endif()
@@ -319,11 +329,6 @@ if (WINGETOPT_FOUND)
   target_link_libraries_system(pow-miner wingetopt)
 endif()
 
-add_library(ton_block ${BLOCK_SOURCE})
-target_include_directories(ton_block PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/block> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
-target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api)
-
 set(TURN_OFF_LSAN cd .)
 if (TON_USE_ASAN AND NOT WIN32)
   set(TURN_OFF_LSAN export LSAN_OPTIONS=detect_leaks=0)

crypto/block/mc-config.cpp

@@ -2100,7 +2100,7 @@ Ref<vm::Cell> ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const {
     return {};
   }
   auto csr = libraries_dict_->lookup(root_hash, 256);
-  if (csr.is_null() || csr->prefetch_ulong(8) != 0 || !csr->have_refs()) {  // shared_lib_descr$00 lib:^Cell
+  if (csr.is_null() || csr->prefetch_ulong(2) != 0 || !csr->have_refs()) {  // shared_lib_descr$00 lib:^Cell
     return {};
   }
   auto lib = csr->prefetch_ref();

crypto/block/transaction.cpp

@@ -271,7 +271,7 @@ bool Account::recompute_tmp_addr(Ref<vm::CellSlice>& tmp_addr, int split_depth,
 }
 
 bool Account::init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite) {
-  if (split_depth_set_ || !created || !set_split_depth(split_depth)) {
+  if (split_depth_set_ || !set_split_depth(split_depth)) {
     return false;
   }
   addr_orig = addr;
@@ -304,15 +304,8 @@ bool Account::unpack(Ref<vm::CellSlice> shard_account, Ref<vm::CellSlice> extra,
   total_state = orig_total_state = account;
   auto acc_cs = load_cell_slice(std::move(account));
   if (block::gen::t_Account.get_tag(acc_cs) == block::gen::Account::account_none) {
-    status = acc_nonexist;
-    last_paid = 0;
-    last_trans_end_lt_ = 0;
     is_special = special;
-    if (workchain != ton::workchainInvalid) {
+    return acc_cs.size_ext() == 1 && init_new(now);
-      addr_orig = addr;
-      addr_rewrite = addr.cbits();
-    }
-    return compute_my_addr() && acc_cs.size_ext() == 1;
   }
   block::gen::Account::Record_account acc;
   block::gen::AccountStorage::Record storage;
@@ -328,6 +321,7 @@ bool Account::unpack(Ref<vm::CellSlice> shard_account, Ref<vm::CellSlice> extra,
     case block::gen::AccountState::account_uninit:
       status = orig_status = acc_uninit;
       state_hash = addr;
+      forget_split_depth();
       break;
     case block::gen::AccountState::account_frozen:
       status = orig_status = acc_frozen;
@@ -396,10 +390,42 @@ bool Account::init_new(ton::UnixTime now) {
   state_hash = addr_orig;
   status = orig_status = acc_nonexist;
   split_depth_set_ = false;
-  created = true;
   return true;
 }
 
+bool Account::forget_split_depth() {
+  split_depth_set_ = false;
+  split_depth_ = 0;
+  addr_orig = addr;
+  my_addr = my_addr_exact;
+  addr_rewrite = addr.bits();
+  return true;
+}
+
+bool Account::deactivate() {
+  if (status == acc_active) {
+    return false;
+  }
+  // forget special (tick/tock) info
+  tick = tock = false;
+  if (status == acc_nonexist || status == acc_uninit) {
+    // forget split depth and address rewriting info
+    forget_split_depth();
+    // forget specific state hash for deleted or uninitialized accounts (revert to addr)
+    state_hash = addr;
+  }
+  // forget code and data (only active accounts remember these)
+  code.clear();
+  data.clear();
+  library.clear();
+  // if deleted, balance must be zero
+  if (status == acc_nonexist && !balance.is_zero()) {
+    return false;
+  }
+  return true;
+}
+
+
 bool Account::belongs_to_shard(ton::ShardIdFull shard) const {
   return workchain == shard.workchain && ton::shard_is_ancestor(shard.shard, addr);
 }
@@ -2214,7 +2240,7 @@ Ref<vm::Cell> Transaction::commit(Account& acc) {
   CHECK((const void*)&acc == (const void*)&account);
   // export all fields modified by the Transaction into original account
   // NB: this is the only method that modifies account
-  if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status == Account::acc_nonexist &&
+  if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status != Account::acc_active &&
       acc_status == Account::acc_active) {
     LOG(DEBUG) << "setting address rewriting info for newly-activated account " << acc.addr.to_hex()
                << " with split_depth=" << new_split_depth
@@ -2243,9 +2269,7 @@ Ref<vm::Cell> Transaction::commit(Account& acc) {
     acc.tick = new_tick;
     acc.tock = new_tock;
   } else {
-    acc.tick = acc.tock = false;
+    CHECK(acc.deactivate());
-    acc.split_depth_set_ = false;
-    acc.created = true;
   }
   end_lt = 0;
   acc.push_transaction(root, start_lt);

crypto/block/transaction.h

@@ -213,17 +213,16 @@ struct Account {
   bool is_special{false};
   bool tick{false};
   bool tock{false};
-  bool created{false};
   bool split_depth_set_{false};
   unsigned char split_depth_{0};
   int verbosity{3 * 0};
   ton::UnixTime now_{0};
   ton::WorkchainId workchain{ton::workchainInvalid};
   td::BitArray<32> addr_rewrite;     // rewrite (anycast) data, split_depth bits
-  ton::StdSmcAddress addr;           // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`)
+  ton::StdSmcAddress addr;           // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`); it is the key in ShardAccounts
-  ton::StdSmcAddress addr_orig;      // address indicated in smart-contract data
+  ton::StdSmcAddress addr_orig;      // address indicated in smart-contract data (must coincide with hash of StateInit)
-  Ref<vm::CellSlice> my_addr;        // address as stored in the smart contract (MsgAddressInt)
+  Ref<vm::CellSlice> my_addr;        // address as stored in the smart contract (MsgAddressInt); corresponds to `addr_orig` + anycast info
-  Ref<vm::CellSlice> my_addr_exact;  // exact address without anycast info
+  Ref<vm::CellSlice> my_addr_exact;  // exact address without anycast info; corresponds to `addr` and has no anycast (rewrite) info
   ton::LogicalTime last_trans_end_lt_;
   ton::LogicalTime last_trans_lt_;
   ton::Bits256 last_trans_hash_;
@@ -250,6 +249,7 @@ struct Account {
   bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr);
   bool unpack(Ref<vm::CellSlice> account, Ref<vm::CellSlice> extra, ton::UnixTime now, bool special = false);
   bool init_new(ton::UnixTime now);
+  bool deactivate();
   bool recompute_tmp_addr(Ref<vm::CellSlice>& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const;
   td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector<block::StoragePrices>& pricing) const;
   bool is_masterchain() const {
@@ -268,6 +268,7 @@ struct Account {
   friend struct Transaction;
   bool set_split_depth(int split_depth);
   bool check_split_depth(int split_depth) const;
+  bool forget_split_depth();
   bool init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite);
 
  private:

crypto/common/bigint.hpp

@@ -264,7 +264,7 @@ class AnyIntView {
     return digits[size() - 1];
   }
   double top_double() const {
-    return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * (1.0 / Tr::Base)
+    return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * Tr::InvBase
                       : (double)digits[size() - 1];
   }
   bool is_odd_any() const {
@@ -314,8 +314,15 @@ class BigIntG {
     digits[0] = x;
   }
   BigIntG(Normalize, word_t x) : n(1) {
+    if (x >= -Tr::Half && x < Tr::Half) {
+      digits[0] = x;
+    } else if (len <= 1) {
       digits[0] = x;
       normalize_bool();
+    } else {
+      digits[0] = ((x + Tr::Half) & (Tr::Base - 1)) - Tr::Half;
+      digits[n++] = (x >> Tr::word_shift) + (digits[0] < 0);
+    }
   }
   BigIntG(const BigIntG& x) : n(x.n) {
     std::memcpy(digits, x.digits, n * sizeof(word_t));
@@ -757,7 +764,7 @@ bool AnyIntView<Tr>::add_pow2_any(int exponent, int factor) {
   while (size() <= k) {
     digits[inc_size()] = 0;
   }
-  digits[k] += (factor << dm.rem);
+  digits[k] += ((word_t)factor << dm.rem);
   return true;
 }
 
@@ -1087,12 +1094,16 @@ int AnyIntView<Tr>::cmp_any(const AnyIntView<Tr>& yp) const {
 
 template <class Tr>
 int AnyIntView<Tr>::cmp_any(word_t y) const {
-  if (size() > 1) {
+  if (size() == 1) {
-    return top_word() < 0 ? -1 : 1;
-  } else if (size() == 1) {
     return digits[0] < y ? -1 : (digits[0] > y ? 1 : 0);
-  } else {
+  } else if (!size()) {
     return 0x80000000;
+  } else if (size() == 2 && (y >= Tr::Half || y < -Tr::Half)) {
+    word_t x0 = digits[0] & (Tr::Base - 1), y0 = y & (Tr::Base - 1);
+    word_t x1 = digits[1] + (digits[0] >> Tr::word_shift), y1 = (y >> Tr::word_shift);
+    return x1 < y1 ? -1 : (x1 > y1 ? 1 : (x0 < y0 ? -1 : (x0 > y0 ? 1 : 0)));
+  } else {
+    return top_word() < 0 ? -1 : 1;
   }
 }
 
@@ -1312,17 +1323,14 @@ bool AnyIntView<Tr>::mod_div_any(const AnyIntView<Tr>& yp, AnyIntView<Tr>& quot,
       if (k > quot.max_size()) {
         return invalidate_bool();
       }
-      quot.set_size(max(k,1));
+      quot.set_size(std::max(k, 1));
-      for(int qi=0; qi< max(k,1); qi++) {
+      quot.digits[0] = 0;
-        quot.digits[qi]=0;
-      }
     } else {
       if (k >= quot.max_size()) {
         return invalidate_bool();
       }
       quot.set_size(k + 1);
-      double x_top = top_double();
+      word_t q = std::llrint(top_double() * y_inv * Tr::InvBase);
-      word_t q = std::llrint(x_top * y_inv * Tr::InvBase);
       quot.digits[k] = q;
       int i = yp.size() - 1;
       word_t hi = 0;
@@ -1337,8 +1345,7 @@ bool AnyIntView<Tr>::mod_div_any(const AnyIntView<Tr>& yp, AnyIntView<Tr>& quot,
     quot.digits[0] = 0;
   }
   while (--k >= 0) {
-    double x_top = top_double();
+    word_t q = std::llrint(top_double() * y_inv);
-    word_t q = std::llrint(x_top * y_inv);
     quot.digits[k] = q;
     for (int i = yp.size() - 1; i >= 0; --i) {
       Tr::sub_mul(&digits[k + i + 1], &digits[k + i], q, yp.digits[i]);
@@ -1346,15 +1353,18 @@ bool AnyIntView<Tr>::mod_div_any(const AnyIntView<Tr>& yp, AnyIntView<Tr>& quot,
     dec_size();
     digits[size() - 1] += (digits[size()] << word_shift);
   }
-  if (size() >= yp.size()) {
+  if (size() >= yp.size() - 1) {
-    assert(size() == yp.size());
+    assert(size() <= yp.size());
-    double x_top = top_double();
+    bool grow = (size() < yp.size());
-    double t = x_top * y_inv * Tr::InvBase;
+    double t = top_double() * y_inv * (grow ? Tr::InvBase * Tr::InvBase : Tr::InvBase);
     if (round_mode >= 0) {
       t += (round_mode ? 1 : 0.5);
     }
     word_t q = std::llrint(std::floor(t));
     if (q) {
+      if (grow) {
+        digits[inc_size()] = 0;
+      }
       for (int i = 0; i < size(); i++) {
         digits[i] -= q * yp.digits[i];
       }
@@ -1411,6 +1421,7 @@ bool AnyIntView<Tr>::mod_div_any(const AnyIntView<Tr>& yp, AnyIntView<Tr>& quot,
   return normalize_bool_any();
 }
 
+// works for almost-normalized numbers (digits -Base+1 .. Base-1, top non-zero), result also almost-normalized
 template <class Tr>
 bool AnyIntView<Tr>::mod_pow2_any(int exponent) {
   if (!is_valid()) {
@@ -1462,25 +1473,21 @@ bool AnyIntView<Tr>::mod_pow2_any(int exponent) {
     if (exponent >= max_size() * word_shift) {
       return invalidate_bool();
     }
-    if (q - word_shift >= 0) {
+    if (q - word_shift >= 0) {  // original top digit was a non-zero multiple of Base, impossible(?)
       digits[size() - 1] = 0;
       digits[inc_size()] = ((word_t)1 << (q - word_shift));
-    }
+    } else if (q - word_shift == -1 && size() < max_size()) {
-    if (q - word_shift == -1 && size() < max_size() - 1) {
       digits[size() - 1] = -Tr::Half;
       digits[inc_size()] = 1;
     } else {
       digits[size() - 1] = pow;
     }
     return true;
-  } else if (v >= Tr::Half) {
+  } else if (v >= Tr::Half && size() < max_size()) {
-    if (size() == max_size() - 1) {
+    word_t w = (((v >> (word_shift - 1)) + 1) >> 1);
-      return invalidate_bool();
+    digits[size() - 1] = v - (w << word_shift);
-    } else {
+    digits[inc_size()] = w;
-      digits[size() - 1] = v | -Tr::Half;
-      digits[inc_size()] = ((word_t)1 << (q - word_shift));
     return true;
-    }
   } else {
     digits[size() - 1] = v;
     return true;

crypto/common/refint.cpp

@@ -128,12 +128,10 @@ RefInt256 muldiv(RefInt256 x, RefInt256 y, RefInt256 z, int round_mode) {
 }
 
 std::pair<RefInt256, RefInt256> muldivmod(RefInt256 x, RefInt256 y, RefInt256 z, int round_mode) {
-  typename td::BigInt256::DoubleInt tmp{0};
+  typename td::BigInt256::DoubleInt tmp{0}, quot;
   tmp.add_mul(*x, *y);
-  RefInt256 quot{true};
+  tmp.mod_div(*z, quot, round_mode);
-  tmp.mod_div(*z, quot.unique_write(), round_mode);
+  return std::make_pair(td::make_refint(quot.normalize()), td::make_refint(tmp));
-  quot.write().normalize();
-  return std::make_pair(std::move(quot), td::make_refint(tmp));
 }
 
 RefInt256 operator&(RefInt256 x, RefInt256 y) {

crypto/fift/lib/Asm.fif

@@ -334,9 +334,14 @@ x{A926} @Defop RSHIFTC
 x{A935} @Defop(8u+1) RSHIFTR#
 x{A936} @Defop(8u+1) RSHIFTC#
 x{A938} @Defop(8u+1) MODPOW2#
+x{A939} @Defop(8u+1) MODPOW2R#
+x{A93A} @Defop(8u+1) MODPOW2C#
 x{A984} @Defop MULDIV
 x{A985} @Defop MULDIVR
+x{A988} @Defop MULMOD
 x{A98C} @Defop MULDIVMOD
+x{A98D} @Defop MULDIVMODR
+x{A98E} @Defop MULDIVMODC
 x{A9A4} @Defop MULRSHIFT
 x{A9A5} @Defop MULRSHIFTR
 x{A9A6} @Defop MULRSHIFTC
@@ -649,7 +654,7 @@ x{DB32} dup @Defop BRANCH @Defop RETBOOL
 x{DB34} @Defop CALLCC
 x{DB35} @Defop JMPXDATA
 { dup 1+ 0= { 16 + } if
-  <b x{DB36} rot 4 u, swap 4 u, @addopb
+  <b x{DB36} s, rot 4 u, swap 4 u, @addopb
 } : CALLCCARGS
 x{DB38} @Defop CALLXVARARGS
 x{DB39} @Defop RETVARARGS
@@ -689,9 +694,9 @@ x{E30F} @Defop(ref*2) IFREFELSEREF
   [] execute
 } : @run-cont-op
 { triple 1 ' @run-cont-op does create } : @def-cont-op
-{ } { PUSHCONT IF } { IFREF } @def-cont-op IF-cont
+{ DROP } { PUSHCONT IF } { IFREF } @def-cont-op IF-cont
 { IFRET } { PUSHCONT IFJMP } { IFJMPREF } @def-cont-op IFJMP-cont
-{ } { PUSHCONT IFNOT } { IFNOTREF } @def-cont-op IFNOT-cont
+{ DROP } { PUSHCONT IFNOT } { IFNOTREF } @def-cont-op IFNOT-cont
 { IFNOTRET } { PUSHCONT IFNOTJMP } { IFNOTJMPREF } @def-cont-op IFNOTJMP-cont
 { dup 2over rot } : 3dup
 
@@ -741,10 +746,10 @@ x{E304} @Defop CONDSEL
 x{E305} @Defop CONDSELCHK
 x{E308} @Defop IFRETALT
 x{E309} @Defop IFNOTRETALT
-{ <b x{E39_} swap 5 u, @addopb } : IFBITJMP
+{ <b x{E39_} s, swap 5 u, @addopb } : IFBITJMP
-{ <b x{E3B_} swap 5 u, @addopb } : IFNBITJMP
+{ <b x{E3B_} s, swap 5 u, @addopb } : IFNBITJMP
-{ <b x{E3D_} swap 5 u, swap ref, @addopb } : IFBITJMPREF
+{ <b x{E3D_} s, swap 5 u, swap ref, @addopb } : IFBITJMPREF
-{ <b x{E3F_} swap 5 u, swap ref, @addopb } : IFNBITJMPREF
+{ <b x{E3F_} s, swap 5 u, swap ref, @addopb } : IFNBITJMPREF
 
 x{E4} @Defop REPEAT
 x{E5} dup @Defop REPEATEND @Defop REPEAT:

crypto/func/abscode.cpp

@@ -168,6 +168,11 @@ void VarDescr::set_const(td::RefInt256 value) {
   }
 }
 
+void VarDescr::set_const(std::string value) {
+  str_const = value;
+  val = _Const;
+}
+
 void VarDescr::set_const_nan() {
   set_const(td::make_refint());
 }
@@ -342,6 +347,11 @@ void Op::show(std::ostream& os, const std::vector<TmpVar>& vars, std::string pfx
       show_var_list(os, left, vars);
       os << " := " << int_const << std::endl;
       break;
+    case _SliceConst:
+      os << pfx << dis << "SCONST ";
+      show_var_list(os, left, vars);
+      os << " := " << str_const << std::endl;
+      break;
     case _Import:
       os << pfx << dis << "IMPORT ";
       show_var_list(os, left, vars);

crypto/func/analyzer.cpp

@@ -353,6 +353,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) {
   }
   switch (cl) {
     case _IntConst:
+    case _SliceConst:
     case _GlobVar:
     case _Call:
     case _CallInd:
@@ -540,6 +541,7 @@ bool prune_unreachable(std::unique_ptr<Op>& ops) {
   bool reach;
   switch (op.cl) {
     case Op::_IntConst:
+    case Op::_SliceConst:
     case Op::_GlobVar:
     case Op::_SetGlob:
     case Op::_Call:
@@ -707,6 +709,10 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
       values.add_newval(left[0]).set_const(int_const);
       break;
     }
+    case _SliceConst: {
+      values.add_newval(left[0]).set_const(str_const);
+      break;
+    }
     case _Call: {
       prepare_args(values);
       auto func = dynamic_cast<const SymValAsmFunc*>(fun_ref->value);
@@ -848,6 +854,7 @@ bool Op::mark_noreturn() {
       // fallthrough
     case _Import:
     case _IntConst:
+    case _SliceConst:
     case _Let:
     case _Tuple:
     case _UnTuple:

crypto/func/asmops.cpp

@@ -55,10 +55,10 @@ std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg) {
   }
 }
 
-AsmOp AsmOp::Const(int arg, std::string push_op) {
+AsmOp AsmOp::Const(int arg, std::string push_op, td::RefInt256 origin) {
   std::ostringstream os;
   os << arg << ' ' << push_op;
-  return AsmOp::Const(os.str());
+  return AsmOp::Const(os.str(), origin);
 }
 
 AsmOp AsmOp::make_stk2(int a, int b, const char* str, int delta) {
@@ -166,27 +166,27 @@ AsmOp AsmOp::UnTuple(int a) {
 
 AsmOp AsmOp::IntConst(td::RefInt256 x) {
   if (x->signed_fits_bits(8)) {
-    return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT");
+    return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x);
   }
   if (!x->is_valid()) {
-    return AsmOp::Const("PUSHNAN");
+    return AsmOp::Const("PUSHNAN", x);
   }
   int k = is_pos_pow2(x);
   if (k >= 0) {
-    return AsmOp::Const(k, "PUSHPOW2");
+    return AsmOp::Const(k, "PUSHPOW2", x);
   }
   k = is_pos_pow2(x + 1);
   if (k >= 0) {
-    return AsmOp::Const(k, "PUSHPOW2DEC");
+    return AsmOp::Const(k, "PUSHPOW2DEC", x);
   }
   k = is_pos_pow2(-x);
   if (k >= 0) {
-    return AsmOp::Const(k, "PUSHNEGPOW2");
+    return AsmOp::Const(k, "PUSHNEGPOW2", x);
   }
   if (!x->mod_pow2_short(23)) {
-    return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX");
+    return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX", x);
   }
-  return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT");
+  return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x);
 }
 
 AsmOp AsmOp::BoolConst(bool f) {

crypto/func/builtins.cpp

@@ -27,7 +27,7 @@ using namespace std::literals::string_literals;
  * 
  */
 
-int glob_func_cnt, undef_func_cnt, glob_var_cnt;
+int glob_func_cnt, undef_func_cnt, glob_var_cnt, const_cnt;
 std::vector<SymDef*> glob_func, glob_vars;
 
 SymDef* predefine_builtin_func(std::string name, TypeExpr* func_type) {

crypto/func/codegen.cpp

@@ -302,6 +302,15 @@ bool Op::generate_code_step(Stack& stack) {
       }
       return true;
     }
+    case _SliceConst: {
+      auto p = next_var_info[left[0]];
+      if (!p || p->is_unused()) {
+        return true;
+      }
+      stack.o << AsmOp::Const("x{" + str_const + "} PUSHSLICE");
+      stack.push_new_var(left[0]);
+      return true;
+    }
     case _GlobVar:
       if (dynamic_cast<const SymValGlobVar*>(fun_ref->value)) {
         bool used = false;

crypto/func/func.cpp

@@ -173,16 +173,29 @@ void usage(const char* progname) {
          "-R\tInclude operation rewrite comments in the output code\n"
          "-W<output-boc-file>\tInclude Fift code to serialize and save generated code into specified BoC file. Enables "
          "-A and -P.\n"
+         "\t-s\tOutput semantic version of FunC and exit\n"
          "\t-V<version>\tShow func build information\n";
   std::exit(2);
 }
 
+void output_inclusion_stack() {
+  while (!funC::inclusion_locations.empty()) {
+    src::SrcLocation loc = funC::inclusion_locations.top();
+    funC::inclusion_locations.pop();
+    if (loc.fdescr) {
+      std::cerr << "note: included from ";
+      loc.show(std::cerr);
+      std::cerr << std::endl;
+    }
+  }
+}
+
 std::string output_filename;
 
 int main(int argc, char* const argv[]) {
   int i;
   bool interactive = false;
-  while ((i = getopt(argc, argv, "Ahi:Io:O:PRSvW:V")) != -1) {
+  while ((i = getopt(argc, argv, "Ahi:Io:O:PRsSvW:V")) != -1) {
     switch (i) {
       case 'A':
         funC::asm_preamble = true;
@@ -215,8 +228,13 @@ int main(int argc, char* const argv[]) {
         funC::boc_output_filename = optarg;
         funC::asm_preamble = funC::program_envelope = true;
         break;
+      case 's':
+        std::cout << funC::func_version << "\n";
+        std::exit(0);
+        break;
       case 'V':
-        std::cout << "Func build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n";
+        std::cout << "FunC semantic version: v" << funC::func_version << "\n";
+        std::cout << "Build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n";
         std::exit(0);
         break;
       case 'h':
@@ -235,7 +253,7 @@ int main(int argc, char* const argv[]) {
   int ok = 0, proc = 0;
   try {
     while (optind < argc) {
-      funC::generated_from += std::string{"`"} + argv[optind] + "` ";
+      // funC::generated_from += std::string{"`"} + argv[optind] + "` ";
       ok += funC::parse_source_file(argv[optind++]);
       proc++;
     }
@@ -262,14 +280,17 @@ int main(int argc, char* const argv[]) {
     funC::generate_output();
   } catch (src::Fatal& fatal) {
     std::cerr << "fatal: " << fatal << std::endl;
+    output_inclusion_stack();
     std::exit(1);
   } catch (src::Error& error) {
     std::cerr << error << std::endl;
+    output_inclusion_stack();
     std::exit(1);
   } catch (funC::UnifyError& unif_err) {
     std::cerr << "fatal: ";
     unif_err.print_message(std::cerr);
     std::cerr << std::endl;
+    output_inclusion_stack();
     std::exit(1);
   }
 }

crypto/func/func.h

@@ -35,9 +35,12 @@ namespace funC {
 
 extern int verbosity;
 extern bool op_rewrite_comments;
+extern std::string generated_from;
 
 constexpr int optimize_depth = 20;
 
+const std::string func_version{"0.2.0"};
+
 enum Keyword {
   _Eof = -1,
   _Ident = 0,
@@ -106,7 +109,10 @@ enum Keyword {
   _Operator,
   _Infix,
   _Infixl,
-  _Infixr
+  _Infixr,
+  _Const,
+  _PragmaHashtag,
+  _IncludeHashtag
 };
 
 void define_keywords();
@@ -333,6 +339,8 @@ struct VarDescr {
   static constexpr int FiniteUInt = FiniteInt | _Pos;
   int val;
   td::RefInt256 int_const;
+  std::string str_const;
+
   VarDescr(var_idx_t _idx = -1, int _flags = 0, int _val = 0) : idx(_idx), flags(_flags), val(_val) {
   }
   bool operator<(var_idx_t other_idx) const {
@@ -403,6 +411,7 @@ struct VarDescr {
   }
   void set_const(long long value);
   void set_const(td::RefInt256 value);
+  void set_const(std::string value);
   void set_const_nan();
   void operator+=(const VarDescr& y) {
     flags &= y.flags;
@@ -527,7 +536,8 @@ struct Op {
     _While,
     _Until,
     _Repeat,
-    _Again
+    _Again,
+    _SliceConst
   };
   int cl;
   enum { _Disabled = 1, _Reachable = 2, _NoReturn = 4, _ImpureR = 8, _ImpureW = 16, _Impure = 24 };
@@ -540,6 +550,7 @@ struct Op {
   std::vector<var_idx_t> left, right;
   std::unique_ptr<Op> block0, block1;
   td::RefInt256 int_const;
+  std::string str_const;
   Op(const SrcLocation& _where = {}, int _cl = _Undef) : cl(_cl), flags(0), fun_ref(nullptr), where(_where) {
   }
   Op(const SrcLocation& _where, int _cl, const std::vector<var_idx_t>& _left)
@@ -551,6 +562,9 @@ struct Op {
   Op(const SrcLocation& _where, int _cl, const std::vector<var_idx_t>& _left, td::RefInt256 _const)
       : cl(_cl), flags(0), fun_ref(nullptr), where(_where), left(_left), int_const(_const) {
   }
+  Op(const SrcLocation& _where, int _cl, const std::vector<var_idx_t>& _left, std::string _const)
+      : cl(_cl), flags(0), fun_ref(nullptr), where(_where), left(_left), str_const(_const) {
+  }
   Op(const SrcLocation& _where, int _cl, const std::vector<var_idx_t>& _left, const std::vector<var_idx_t>& _right,
      SymDef* _fun = nullptr)
       : cl(_cl), flags(0), fun_ref(_fun), where(_where), left(_left), right(_right) {
@@ -781,6 +795,30 @@ struct SymValGlobVar : sym::SymValBase {
   }
 };
 
+struct SymValConst : sym::SymValBase {
+  td::RefInt256 intval;
+  std::string strval;
+  Keyword type;
+  SymValConst(int idx, td::RefInt256 value)
+      : sym::SymValBase(_Const, idx), intval(value) {
+    type = _Int;
+  }
+  SymValConst(int idx, std::string value)
+      : sym::SymValBase(_Const, idx), strval(value) {
+    type = _Slice;
+  }
+  ~SymValConst() override = default;
+  td::RefInt256 get_int_value() const {
+    return intval;
+  }
+  std::string get_str_value() const {
+    return strval;
+  }
+  Keyword get_type() const {
+    return type;
+  }
+};
+
 extern int glob_func_cnt, undef_func_cnt, glob_var_cnt;
 extern std::vector<SymDef*> glob_func, glob_vars;
 
@@ -792,9 +830,11 @@ extern std::vector<SymDef*> glob_func, glob_vars;
 
 // defined in parse-func.cpp
 bool parse_source(std::istream* is, const src::FileDescr* fdescr);
-bool parse_source_file(const char* filename);
+bool parse_source_file(const char* filename, src::Lexem lex = {});
 bool parse_source_stdin();
 
+extern std::stack<src::SrcLocation> inclusion_locations;
+
 /*
  * 
  *   EXPRESSIONS
@@ -817,7 +857,8 @@ struct Expr {
     _LetFirst,
     _Hole,
     _Type,
-    _CondExpr
+    _CondExpr,
+    _SliceConst
   };
   int cls;
   int val{0};
@@ -825,6 +866,7 @@ struct Expr {
   int flags{0};
   SrcLocation here;
   td::RefInt256 intval;
+  std::string strval;
   SymDef* sym{nullptr};
   TypeExpr* e_type{nullptr};
   std::vector<Expr*> args;
@@ -907,6 +949,7 @@ struct AsmOp {
   int a, b, c;
   bool gconst{false};
   std::string op;
+  td::RefInt256 origin;
   struct SReg {
     int idx;
     SReg(int _idx) : idx(_idx) {
@@ -926,6 +969,9 @@ struct AsmOp {
   AsmOp(int _t, int _a, int _b, std::string _op) : t(_t), a(_a), b(_b), op(std::move(_op)) {
     compute_gconst();
   }
+  AsmOp(int _t, int _a, int _b, std::string _op, td::RefInt256 x) : t(_t), a(_a), b(_b), op(std::move(_op)), origin(x) {
+    compute_gconst();
+  }
   AsmOp(int _t, int _a, int _b, int _c) : t(_t), a(_a), b(_b), c(_c) {
   }
   AsmOp(int _t, int _a, int _b, int _c, std::string _op) : t(_t), a(_a), b(_b), c(_c), op(std::move(_op)) {
@@ -1044,10 +1090,10 @@ struct AsmOp {
   static AsmOp make_stk3(int a, int b, int c, const char* str, int delta);
   static AsmOp IntConst(td::RefInt256 value);
   static AsmOp BoolConst(bool f);
-  static AsmOp Const(std::string push_op) {
+  static AsmOp Const(std::string push_op, td::RefInt256 origin = {}) {
-    return AsmOp(a_const, 0, 1, std::move(push_op));
+    return AsmOp(a_const, 0, 1, std::move(push_op), origin);
   }
-  static AsmOp Const(int arg, std::string push_op);
+  static AsmOp Const(int arg, std::string push_op, td::RefInt256 origin = {});
   static AsmOp Comment(std::string comment) {
     return AsmOp(a_none, std::string{"// "} + comment);
   }

crypto/func/gen-abscode.cpp

@@ -362,6 +362,11 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
       code.close_pop_cur(args[2]->here);
       return rvect;
     }
+    case _SliceConst: {
+      auto rvect = new_tmp_vect(code);
+      code.emplace_back(here, Op::_SliceConst, rvect, strval);
+      return rvect;
+    }
     default:
       std::cerr << "expression constructor is " << cls << std::endl;
       throw src::Fatal{"cannot compile expression with unknown constructor"};

crypto/func/keywords.cpp

@@ -125,7 +125,11 @@ void define_keywords() {
       .add_keyword("operator", Kw::_Operator)
       .add_keyword("infix", Kw::_Infix)
       .add_keyword("infixl", Kw::_Infixl)
-      .add_keyword("infixr", Kw::_Infixr);
+      .add_keyword("infixr", Kw::_Infixr)
+      .add_keyword("const", Kw::_Const);
+
+  sym::symbols.add_keyword("#pragma", Kw::_PragmaHashtag)
+      .add_keyword("#include", Kw::_IncludeHashtag);
 }
 
 }  // namespace funC

crypto/func/parse-func.cpp

@@ -19,7 +19,11 @@
 #include "func.h"
 #include "td/utils/crypto.h"
 #include "common/refint.h"
+#include "openssl/digest.hpp"
+#include "block/block.h"
+#include "block-parse.h"
 #include <fstream>
+#include "td/utils/port/path.h"
 
 namespace sym {
 
@@ -229,6 +233,83 @@ void parse_global_var_decl(Lexer& lex) {
   lex.next();
 }
 
+extern int const_cnt;
+Expr* parse_expr(Lexer& lex, CodeBlob& code, bool nv = false);
+
+void parse_const_decl(Lexer& lex) {
+  SrcLocation loc = lex.cur().loc;
+  int wanted_type = Expr::_None;
+  if (lex.tp() == _Int) {
+    wanted_type = Expr::_Const;
+    lex.next();
+  } else if (lex.tp() == _Slice) {
+    wanted_type = Expr::_SliceConst;
+    lex.next();
+  }
+  if (lex.tp() != _Ident) {
+    lex.expect(_Ident, "constant name");
+  }
+  loc = lex.cur().loc;
+  SymDef* sym_def = sym::define_global_symbol(lex.cur().val, false, loc);
+  if (!sym_def) {
+    lex.cur().error_at("cannot define global symbol `", "`");
+  }
+  if (sym_def->value) {
+    lex.cur().error_at("global symbol `", "` already exists");
+  }
+  lex.next();
+  if (lex.tp() != '=') {
+    lex.cur().error_at("expected = instead of ", "");
+  }
+  lex.next();
+  CodeBlob code;
+  // Handles processing and resolution of literals and consts
+  auto x = parse_expr(lex, code, false); // also does lex.next() !
+  if (x->flags != Expr::_IsRvalue) {
+    lex.cur().error("expression is not strictly Rvalue");
+  }
+  if ((wanted_type == Expr::_Const) && (x->cls == Expr::_Apply))
+    wanted_type = Expr::_None; // Apply is additionally checked to result in an integer
+  if ((wanted_type != Expr::_None) && (x->cls != wanted_type)) {
+    lex.cur().error("expression type does not match wanted type");
+  }
+  if (x->cls == Expr::_Const) { // Integer constant
+    sym_def->value = new SymValConst{const_cnt++, x->intval};
+  } else if (x->cls == Expr::_SliceConst) { // Slice constant (string)
+    sym_def->value = new SymValConst{const_cnt++, x->strval};
+  } else if (x->cls == Expr::_Apply) {
+    code.emplace_back(loc, Op::_Import, std::vector<var_idx_t>());
+    auto tmp_vars = x->pre_compile(code);
+    code.emplace_back(loc, Op::_Return, std::move(tmp_vars));
+    code.emplace_back(loc, Op::_Nop); // This is neccessary to prevent SIGSEGV!
+    // It is REQUIRED to execute "optimizations" as in func.cpp
+    code.simplify_var_types();
+    code.prune_unreachable_code();
+    code.split_vars(true);
+    for (int i = 0; i < 16; i++) {
+      code.compute_used_code_vars();
+      code.fwd_analyze();
+      code.prune_unreachable_code();
+    }
+    code.mark_noreturn();
+    AsmOpList out_list(0, &code.vars);
+    code.generate_code(out_list);
+    if (out_list.list_.size() != 1) {
+      lex.cur().error("precompiled expression must result in single operation");
+    }
+    auto op = out_list.list_[0];
+    if (!op.is_const()) {
+      lex.cur().error("precompiled expression must result in compilation time constant");
+    }
+    if (op.origin.is_null() || !op.origin->is_valid()) {
+      lex.cur().error("precompiled expression did not result in a valid integer constant");
+    }
+    sym_def->value = new SymValConst{const_cnt++, op.origin};
+  } else {
+    lex.cur().error("integer or slice literal or constant expected");
+  }
+}
+
 FormalArgList parse_formal_args(Lexer& lex) {
   FormalArgList args;
   lex.expect('(', "formal argument list");
@@ -246,6 +327,18 @@ FormalArgList parse_formal_args(Lexer& lex) {
   return args;
 }
 
+void parse_const_decls(Lexer& lex) {
+  lex.expect(_Const);
+  while (true) {
+    parse_const_decl(lex);
+    if (lex.tp() != ',') {
+      break;
+    }
+    lex.expect(',');
+  }
+  lex.expect(';');
+}
+
 TypeExpr* extract_total_arg_type(const FormalArgList& arg_list) {
   if (arg_list.empty()) {
     return TypeExpr::new_unit();
@@ -322,8 +415,6 @@ Expr* make_func_apply(Expr* fun, Expr* x) {
   return res;
 }
 
-Expr* parse_expr(Lexer& lex, CodeBlob& code, bool nv = false);
-
 // parse ( E { , E } ) | () | [ E { , E } ] | [] | id | num | _
 Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) {
   if (lex.tp() == '(' || lex.tp() == '[') {
@@ -388,6 +479,87 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) {
     lex.next();
     return res;
   }
+  if (t == Lexem::String) {
+    std::string str = lex.cur().str;
+    int str_type = lex.cur().val;
+    Expr* res;
+    switch (str_type) {
+      case 0:
+      case 's':
+      case 'a':
+      {
+        res = new Expr{Expr::_SliceConst, lex.cur().loc};
+        res->e_type = TypeExpr::new_atomic(_Slice);
+        break;
+      }
+      case 'u':
+      case 'h':
+      case 'H':
+      case 'c':
+      {
+        res = new Expr{Expr::_Const, lex.cur().loc};
+        res->e_type = TypeExpr::new_atomic(_Int);
+        break;
+      }
+      default:
+      {
+        res = new Expr{Expr::_Const, lex.cur().loc};
+        res->e_type = TypeExpr::new_atomic(_Int);
+        lex.cur().error("invalid string type `" + std::string(1, static_cast<char>(str_type)) + "`");
+        return res;
+      }
+    }
+    res->flags = Expr::_IsRvalue;
+    switch (str_type) {
+      case 0: {
+        res->strval = td::hex_encode(str);
+        break;
+      }
+      case 's': {
+        res->strval = str;
+        unsigned char buff[128];
+        int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size());
+        if (bits < 0) {
+          lex.cur().error_at("Invalid hex bitstring constant `", "`");
+        }
+        break;
+      }
+      case 'a': {  // MsgAddressInt
+        block::StdAddress a;
+        if (a.parse_addr(str)) {
+          res->strval = block::tlb::MsgAddressInt().pack_std_address(a)->as_bitslice().to_hex();
+        } else {
+          lex.cur().error_at("invalid standard address `", "`");
+        }
+        break;
+      }
+      case 'u': {
+        res->intval = td::hex_string_to_int256(td::hex_encode(str));
+        if (!str.size()) {
+          lex.cur().error("empty integer ascii-constant");
+        }
+        if (res->intval.is_null()) {
+          lex.cur().error_at("too long integer ascii-constant `", "`");
+        }
+        break;
+      }
+      case 'h':
+      case 'H':
+      {
+        unsigned char hash[32];
+        digest::hash_str<digest::SHA256>(hash, str.data(), str.size());
+        res->intval = td::bits_to_refint(hash, (str_type == 'h') ? 32 : 256, false);
+        break;
+      }
+      case 'c':
+      {
+        res->intval = td::make_refint(td::crc32(td::Slice{str}));
+        break;
+      }
+    }
+    lex.next();
+    return res;
+  }
   if (t == '_') {
     Expr* res = new Expr{Expr::_Hole, lex.cur().loc};
     res->val = -1;
@@ -429,6 +601,25 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) {
       lex.next();
       return res;
     }
+    if (sym && dynamic_cast<SymValConst*>(sym->value)) {
+      auto val = dynamic_cast<SymValConst*>(sym->value);
+      Expr* res = new Expr{Expr::_None, lex.cur().loc};
+      res->flags = Expr::_IsRvalue;
+      if (val->type == _Int) {
+        res->cls = Expr::_Const;
+        res->intval = val->get_int_value();
+      }
+      else if (val->type == _Slice) {
+        res->cls = Expr::_SliceConst;
+        res->strval = val->get_str_value();
+      }
+      else {
+        lex.cur().error("Invalid symbolic constant type");
+      }
+      res->e_type = TypeExpr::new_atomic(val->type);
+      lex.next();
+      return res;
+    }
     bool auto_apply = false;
     Expr* res = new Expr{Expr::_Var, lex.cur().loc};
     if (nv) {
@@ -1288,14 +1479,168 @@ void parse_func_def(Lexer& lex) {
   sym::close_scope(lex);
 }
 
+std::string func_ver_test = func_version;
+
+void parse_pragma(Lexer& lex) {
+  auto pragma = lex.cur();
+  lex.next();
+  if (lex.tp() != _Ident) {
+    lex.expect(_Ident, "pragma name expected");
+  }
+  auto pragma_name = lex.cur().str;
+  lex.next();
+  if (!pragma_name.compare("version") || !pragma_name.compare("not-version")) {
+    bool negate = !pragma_name.compare("not-version");
+    char op = '='; bool eq = false;
+    int sem_ver[3] = {0, 0, 0};
+    char segs = 1;
+    if (lex.tp() == _Number) {
+      sem_ver[0] = std::stoi(lex.cur().str);
+    } else if (lex.tp() == _Ident) {
+      auto id1 = lex.cur().str;
+      char ch1 = id1[0];
+      if ((ch1 == '>') || (ch1 == '<') || (ch1 == '=') || (ch1 == '^')) {
+        op = ch1;
+      } else {
+        lex.cur().error("unexpected comparator operation");
+      }
+      if (id1.length() < 2) {
+        lex.cur().error("expected number after comparator");
+      }
+      if (id1[1] == '=') {
+        eq = true;
+        if (id1.length() < 3) {
+          lex.cur().error("expected number after comparator");
+        }
+        sem_ver[0] = std::stoi(id1.substr(2));
+      } else {
+        sem_ver[0] = std::stoi(id1.substr(1));
+      }
+    } else {
+      lex.cur().error("expected semver with optional comparator");
+    }
+    lex.next();
+    if (lex.tp() != ';') {
+      if (lex.tp() != _Ident || lex.cur().str[0] != '.') {
+        lex.cur().error("invalid semver format");
+      }
+      sem_ver[1] = std::stoi(lex.cur().str.substr(1));
+      segs = 2;
+      lex.next();
+    }
+    if (lex.tp() != ';') {
+      if (lex.tp() != _Ident || lex.cur().str[0] != '.') {
+        lex.cur().error("invalid semver format");
+      }
+      sem_ver[2] = std::stoi(lex.cur().str.substr(1));
+      segs = 3;
+      lex.next();
+    }
+    // End reading semver from source code
+    int func_ver[3] = {0, 0, 0};
+    std::istringstream iss(func_ver_test);
+    std::string s;
+    for (int idx = 0; idx < 3; idx++) {
+      std::getline(iss, s, '.');
+      func_ver[idx] = std::stoi(s);
+    }
+    // End parsing embedded semver
+    std::string semver_expr;
+    if (negate) {
+      semver_expr += '!';
+    }
+    semver_expr += op;
+    if (eq) {
+      semver_expr += '=';
+    }
+    for (int idx = 0; idx < 3; idx++) {
+      semver_expr += std::to_string(sem_ver[idx]);
+      if (idx < 2)
+        semver_expr += '.';
+    }
+    bool match = true;
+    switch (op) {
+      case '=':
+        if ((func_ver[0] != sem_ver[0]) ||
+            (func_ver[1] != sem_ver[1]) ||
+            (func_ver[2] != sem_ver[2])) {
+          match = false;
+        }
+        break;
+      case '>':
+        if ( ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) ||
+             ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] < sem_ver[2])) ||
+             ((func_ver[0] == sem_ver[0]) && (func_ver[1] < sem_ver[1])) ||
+             ((func_ver[0] < sem_ver[0])) ) {
+          match = false;
+        }
+        break;
+      case '<':
+        if ( ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) ||
+             ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] > sem_ver[2])) ||
+             ((func_ver[0] == sem_ver[0]) && (func_ver[1] > sem_ver[1])) ||
+             ((func_ver[0] > sem_ver[0])) ) {
+          match = false;
+        }
+        break;
+      case '^':
+        if ( ((segs == 3) && ((func_ver[0] != sem_ver[0]) || (func_ver[1] != sem_ver[1]) || (func_ver[2] < sem_ver[2])))
+          || ((segs == 2) && ((func_ver[0] != sem_ver[0]) || (func_ver[1] < sem_ver[1])))
+          || ((segs == 1) && ((func_ver[0] < sem_ver[0]))) ) {
+          match = false;
+        }
+        break;
+    }
+    if ((match && negate) || (!match && !negate)) {
+      pragma.error(std::string("FunC version ") + func_ver_test + " does not satisfy condition " + semver_expr);
+    }
+  } else if (!pragma_name.compare("test-version-set")) {
+    if (lex.tp() != _String) {
+      lex.cur().error("version string expected");
+    }
+    func_ver_test = lex.cur().str;
+    lex.next();
+  } else {
+    lex.cur().error(std::string{"unknown pragma `"} + pragma_name + "`");
+  }
+  lex.expect(';');
+}
+
 std::vector<const src::FileDescr*> source_fdescr;
 
+std::vector<std::string> source_files;
+std::stack<src::SrcLocation> inclusion_locations;
+
+void parse_include(Lexer& lex, const src::FileDescr* fdescr) {
+  auto include = lex.cur();
+  lex.expect(_IncludeHashtag);
+  if (lex.tp() != _String) {
+    lex.expect(_String, "source file name");
+  }
+  std::string val = lex.cur().str;
+  std::string parent_dir = fdescr->filename;
+  if (parent_dir.rfind('/') != std::string::npos) {
+    val = parent_dir.substr(0, parent_dir.rfind('/') + 1) + val;
+  }
+  lex.next();
+  lex.expect(';');
+  if (!parse_source_file(val.c_str(), include)) {
+    include.error(std::string{"failed parsing included file `"} + val + "`");
+  }
+}
+
 bool parse_source(std::istream* is, src::FileDescr* fdescr) {
   src::SourceReader reader{is, fdescr};
   Lexer lex{reader, true, ";,()[] ~."};
   while (lex.tp() != _Eof) {
-    if (lex.tp() == _Global) {
+    if (lex.tp() == _PragmaHashtag) {
+      parse_pragma(lex);
+    } else if (lex.tp() == _IncludeHashtag) {
+      parse_include(lex, fdescr);
+    } else if (lex.tp() == _Global) {
       parse_global_var_decls(lex);
+    } else if (lex.tp() == _Const) {
+      parse_const_decls(lex);
     } else {
       parse_func_def(lex);
     }
@@ -1303,17 +1648,46 @@ bool parse_source(std::istream* is, src::FileDescr* fdescr) {
   return true;
 }
 
-bool parse_source_file(const char* filename) {
+bool parse_source_file(const char* filename, src::Lexem lex) {
   if (!filename || !*filename) {
-    throw src::Fatal{"source file name is an empty string"};
+    auto msg = "source file name is an empty string";
+    if (lex.tp) {
+      lex.error(msg);
+    } else {
+      throw src::Fatal{msg};
     }
+  }
+  std::string real_filename = td::realpath(td::CSlice(filename)).move_as_ok();
+  if (std::count(source_files.begin(), source_files.end(), real_filename)) {
+    if (verbosity >= 2) {
+      if (lex.tp) {
+        lex.loc.show_warning(std::string{"skipping file "} + real_filename + " because it was already included");
+      } else {
+        std::cerr << "warning: skipping file " << real_filename << " because it was already included" << std::endl;
+      }
+    }
+    return true;
+  }
+  if (lex.tp) { // included
+    funC::generated_from += std::string{"incl:"};
+  }
+  funC::generated_from += std::string{"`"} + filename + "` ";
+  source_files.push_back(real_filename);
   src::FileDescr* cur_source = new src::FileDescr{filename};
   source_fdescr.push_back(cur_source);
   std::ifstream ifs{filename};
   if (ifs.fail()) {
-    throw src::Fatal{std::string{"cannot open source file `"} + filename + "`"};
+    auto msg = std::string{"cannot open source file `"} + filename + "`";
+    if (lex.tp) {
+      lex.error(msg);
+    } else {
+      throw src::Fatal{msg};
     }
-  return parse_source(&ifs, cur_source);
+  }
+  inclusion_locations.push(lex.loc);
+  bool res = parse_source(&ifs, cur_source);
+  inclusion_locations.pop();
+  return res;
 }
 
 bool parse_source_stdin() {

crypto/func/test/co1.fc

@@ -0,0 +1,55 @@
+const int1 = 1, int2 = 2;
+
+const int int101 = 101;
+const int int111 = 111;
+
+const int1r = int1;
+
+const str1 = "const1", str2 = "aabbcc"s;
+
+const slice str2r = str2;
+
+const str1int = 0x636f6e737431;
+const str2int = 0xAABBCC;
+
+const int nibbles = 4;
+
+int iget1() { return int1; }
+int iget2() { return int2; }
+int iget3() { return int1 + int2; }
+
+int iget1r() { return int1r; }
+
+slice sget1() { return str1; }
+slice sget2() { return str2; }
+slice sget2r() { return str2r; }
+
+const int int240 = ((int1 + int2) * 10) << 3;
+
+int iget240() { return int240; }
+
+builder newc() asm "NEWC";
+slice endcs(builder b) asm "ENDC" "CTOS";
+int sdeq (slice s1, slice s2) asm "SDEQ";
+builder stslicer(builder b, slice s) asm "STSLICER";
+
+_ main() {
+    int i1 = iget1();
+    int i2 = iget2();
+    int i3 = iget3();
+
+    throw_unless(int101, i1 == 1);
+    throw_unless(102, i2 == 2);
+    throw_unless(103, i3 == 3);
+
+    slice s1 = sget1();
+    slice s2 = sget2();
+    slice s3 = newc().stslicer(str1).stslicer(str2r).endcs();
+
+    throw_unless(int111, sdeq(s1, newc().store_uint(str1int, 12 * nibbles).endcs()));
+    throw_unless(112, sdeq(s2, newc().store_uint(str2int, 6 * nibbles).endcs()));
+    throw_unless(113, sdeq(s3, newc().store_uint(0x636f6e737431ABCDEF, 18 * nibbles).endcs()));
+
+    int i4 = iget240();
+    throw_unless(104, i4 == 240);
+}

crypto/func/test/i1.fc

@@ -0,0 +1,16 @@
+global int i;
+
+#include "i1sub1.fc";
+
+() sub0() impure { i = 0; }
+
+#include "i1sub2.fc";
+
+() main() impure {
+  sub0();
+  sub1();
+  sub2();
+  i = 9;
+}
+
+#include "../test/i1sub2.fc";

crypto/func/test/i1sub1.fc

@@ -0,0 +1,8 @@
+;; DO NOT COMPILE DIRECTLY!
+;; Compile i1.fc
+
+#include "i1sub1.fc";
+
+() sub1() impure {
+  i = 1;
+}

crypto/func/test/i1sub2.fc

@@ -0,0 +1,10 @@
+;; DO NOT COMPILE DIRECTLY!
+;; Compile i1.fc
+
+#include "./i1sub1.fc";
+
+() sub2() impure {
+  sub1();
+  sub0();
+  i = 2;
+}

crypto/func/test/pv.fc

@@ -0,0 +1,53 @@
+#pragma test-version-set "1.2.3";
+
+;; Positive tests
+#pragma version ^1.2.0;
+#pragma version ^1.2.3;
+#pragma version >1.2.0;
+#pragma version >0.9.9;
+#pragma version <1.3.0;
+#pragma version <2.0.0;
+#pragma version >=1.2.0;
+#pragma version <=1.3.0;
+#pragma version >=1.2.3;
+#pragma version <=1.2.3;
+#pragma version ^1.2.3;
+#pragma version 1.2.3;
+#pragma version =1.2.3;
+
+;; Negative tests
+#pragma not-version ^1.1.0;
+#pragma not-version ^1.0.0;
+#pragma not-version ^0.2.3;
+#pragma not-version ^2.2.3;
+#pragma not-version ^1.3.3;
+#pragma not-version >1.2.3;
+#pragma not-version <1.2.3;
+#pragma not-version ^1.2.4;
+#pragma not-version >=1.2.4;
+#pragma not-version <=1.2.2;
+#pragma not-version 3.2.1;
+#pragma not-version =3.2.1;
+
+;; Test incomplete (partial) version
+#pragma version ^1.2;
+#pragma version >1.2;
+#pragma version <1.3;
+#pragma version <2;
+#pragma version >=1.2;
+#pragma version <=1.3;
+
+;; Advanced ^ behaviour (partials)
+#pragma version ^1.2;
+#pragma version ^1.0;
+#pragma version ^1;
+#pragma version ^0;
+#pragma not-version ^1.0.0;
+#pragma not-version ^0.0.0;
+#pragma not-version ^0.0;
+#pragma not-version ^1.3;
+#pragma not-version ^2;
+
+(int) main(int a) {
+    return a;
+}

crypto/func/test/s1.fc

@@ -0,0 +1,49 @@
+slice ascii_slice() method_id {
+  return "string";
+}
+
+slice raw_slice() method_id {
+  return "abcdef"s;
+}
+
+slice addr_slice() method_id {
+  return "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a;
+}
+
+int string_hex() method_id {
+  return "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"u;
+}
+
+int string_minihash() method_id {
+  return "transfer(slice, int)"h;
+}
+
+int string_maxihash() method_id {
+  return "transfer(slice, int)"H;
+}
+
+int string_crc32() method_id {
+  return "transfer(slice, int)"c;
+}
+
+builder newc() asm "NEWC";
+slice endcs(builder b) asm "ENDC" "CTOS";
+int sdeq (slice s1, slice s2) asm "SDEQ";
+
+_ main() {
+  slice s_ascii = ascii_slice();
+  slice s_raw = raw_slice();
+  slice s_addr = addr_slice();
+  int i_hex = string_hex();
+  int i_mini = string_minihash();
+  int i_maxi = string_maxihash();
+  int i_crc = string_crc32();
+  throw_unless(101, sdeq(s_ascii, newc().store_uint(0x737472696E67, 12 * 4).endcs()));
+  throw_unless(102, sdeq(s_raw, newc().store_uint(0xABCDEF, 6 * 4).endcs()));
+  throw_unless(103, sdeq(s_addr, newc().store_uint(4, 3).store_int(-1, 8)
+    .store_uint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs()));
+  throw_unless(104, i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435);
+  throw_unless(105, i_mini == 0x7a62e8a8);
+  throw_unless(106, i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979);
+  throw_unless(107, i_crc == 2235694568);
+}

crypto/parser/lexer.cpp

@@ -247,6 +247,11 @@ const Lexem& Lexer::next() {
     }
     lexem.set(std::string{src.get_ptr() + 1, end}, src.here(), qc == '`' ? Lexem::Unknown : Lexem::String);
     src.set_ptr(end + 1);
+    c = src.cur_char();
+    if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
+      lexem.val = c;
+      src.set_ptr(end + 2);
+    }
     // std::cerr << lexem.name_str() << ' ' << lexem.str << std::endl;
     return lexem;
   }

crypto/parser/symtable.h

@@ -32,7 +32,7 @@ namespace sym {
 typedef int var_idx_t;
 
 struct SymValBase {
-  enum { _Param, _Var, _Func, _Typename, _GlobVar };
+  enum { _Param, _Var, _Func, _Typename, _GlobVar, _Const };
   int type;
   int idx;
   SymValBase(int _type, int _idx) : type(_type), idx(_idx) {

crypto/smc-envelope/SmartContract.cpp

@@ -52,30 +52,51 @@ td::Ref<vm::Stack> prepare_vm_stack(td::RefInt256 amount, td::Ref<vm::CellSlice>
   return stack_ref;
 }
 
-td::Ref<vm::Tuple> prepare_vm_c7(td::uint32 now, td::uint64 balance) {
+td::Ref<vm::Tuple> prepare_vm_c7(SmartContract::Args args) {
-  // TODO: fix initialization of c7
   td::BitArray<256> rand_seed;
   rand_seed.as_slice().fill(0);
   td::RefInt256 rand_seed_int{true};
   rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false);
+
+  td::uint32 now = 0;
+  if (args.now) {
+    now = args.now.unwrap();
+  }
+
+  vm::CellBuilder cb;
+  if (args.address) {
+    td::BigInt256 dest_addr;
+    dest_addr.import_bits((*args.address).addr.as_bitslice());
+    cb.store_ones(1)
+        .store_zeroes(2)
+        .store_long((*args.address).workchain, 8)
+        .store_int256(dest_addr, 256);
+  }
+  auto address = cb.finalize();
+  auto config = td::Ref<vm::Cell>();
+
+  if (args.config) {
+    config = (*args.config)->get_root_cell();
+  }
+
   auto tuple = vm::make_tuple_ref(
       td::make_refint(0x076ef1ea),                           // [ magic:0x076ef1ea
       td::make_refint(0),                                    //   actions:Integer
       td::make_refint(0),                                    //   msgs_sent:Integer
       td::make_refint(now),                                  //   unixtime:Integer
-      td::make_refint(0),                                    //   block_lt:Integer
+      td::make_refint(0),             //TODO:                //   block_lt:Integer
-      td::make_refint(0),                                    //   trans_lt:Integer
+      td::make_refint(0),             //TODO:                //   trans_lt:Integer
       std::move(rand_seed_int),                              //   rand_seed:Integer
-      block::CurrencyCollection(balance).as_vm_tuple(),      //   balance_remaining:[Integer (Maybe Cell)]
+      block::CurrencyCollection(args.balance).as_vm_tuple(),      //   balance_remaining:[Integer (Maybe Cell)]
-      vm::load_cell_slice_ref(vm::CellBuilder().finalize())  //  myself:MsgAddressInt
+      vm::load_cell_slice_ref(address),  //  myself:MsgAddressInt
-                                                             //vm::StackEntry::maybe(td::Ref<vm::Cell>())
+      vm::StackEntry::maybe(config)       //vm::StackEntry::maybe(td::Ref<vm::Cell>())
   );                                                         //  global_config:(Maybe Cell) ] = SmartContractInfo;
   //LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string();
   return vm::make_tuple_ref(std::move(tuple));
 }
 
 SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref<vm::Stack> stack, td::Ref<vm::Tuple> c7,
-                                    vm::GasLimits gas, bool ignore_chksig) {
+                                    vm::GasLimits gas, bool ignore_chksig, td::Ref<vm::Cell> libraries) {
   auto gas_credit = gas.gas_credit;
   vm::init_op_cp0();
   vm::DictionaryBase::get_empty_dictionary();
@@ -108,11 +129,15 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref<vm::Stac
   vm::VmState vm{state.code, std::move(stack), gas, 1, state.data, log};
   vm.set_c7(std::move(c7));
   vm.set_chksig_always_succeed(ignore_chksig);
+  if (!libraries.is_null()) {
+    vm.register_library_collection(libraries);
+  }
   try {
     res.code = ~vm.run();
   } catch (...) {
     LOG(FATAL) << "catch unhandled exception";
   }
+  td::ConstBitPtr mlib = vm.get_missing_library();
   res.new_state = std::move(state);
   res.stack = vm.get_stack_ref();
   gas = vm.get_gas_limits();
@@ -128,13 +153,17 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref<vm::Stac
     LOG(DEBUG) << "VM accepted: " << res.accepted;
     LOG(DEBUG) << "VM success: " << res.success;
   }
+  if (!mlib.is_null()) {
+    LOG(DEBUG) << "Missing library: " << mlib.to_hex(256);
+    res.missing_library = mlib;
+  }
   if (res.success) {
     res.new_state.data = vm.get_c4();
     res.actions = vm.get_d(5);
     LOG(DEBUG) << "output actions:\n"
                << block::gen::OutList{res.output_actions_count(res.actions)}.as_string_ref(res.actions);
   }
-  LOG_IF(ERROR, gas_credit != 0 && (res.accepted && !res.success))
+  LOG_IF(ERROR, gas_credit != 0 && (res.accepted && !res.success) && mlib.is_null())
       << "Accepted but failed with code " << res.code << "\n"
       << res.gas_used << "\n";
   return res;
@@ -176,12 +205,8 @@ td::Ref<vm::Cell> SmartContract::get_init_state() const {
 }
 
 SmartContract::Answer SmartContract::run_method(Args args) {
-  td::uint32 now = 0;
-  if (args.now) {
-    now = args.now.unwrap();
-  }
   if (!args.c7) {
-    args.c7 = prepare_vm_c7(now, args.balance);
+    args.c7 = prepare_vm_c7(args);
   }
   if (!args.limits) {
     bool is_internal = args.get_method_id().ok() == 0;
@@ -193,28 +218,26 @@ SmartContract::Answer SmartContract::run_method(Args args) {
   CHECK(args.method_id);
   args.stack.value().write().push_smallint(args.method_id.unwrap());
   auto res =
-      run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig);
+      run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig,
+                    args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref<vm::Cell>{});
   state_ = res.new_state;
   return res;
 }
 
 SmartContract::Answer SmartContract::run_get_method(Args args) const {
-  td::uint32 now = 0;
-  if (args.now) {
-    now = args.now.unwrap();
-  }
   if (!args.c7) {
-    args.c7 = prepare_vm_c7(now, args.balance);
+    args.c7 = prepare_vm_c7(args);
   }
   if (!args.limits) {
-    args.limits = vm::GasLimits{1000000};
+    args.limits = vm::GasLimits{1000000, 1000000};
   }
   if (!args.stack) {
     args.stack = td::Ref<vm::Stack>(true);
   }
   CHECK(args.method_id);
   args.stack.value().write().push_smallint(args.method_id.unwrap());
-  return run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig);
+  return run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig,
+                       args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref<vm::Cell>{});
 }
 
 SmartContract::Answer SmartContract::run_get_method(td::Slice method, Args args) const {

crypto/smc-envelope/SmartContract.h

@@ -26,6 +26,7 @@
 #include "td/utils/crypto.h"
 
 #include "block/block.h"
+#include "block/mc-config.h"
 
 namespace ton {
 class SmartContract : public td::CntObject {
@@ -48,6 +49,7 @@ class SmartContract : public td::CntObject {
     td::Ref<vm::Cell> actions;
     td::int32 code;
     td::int64 gas_used;
+    td::ConstBitPtr missing_library{0};
     static int output_actions_count(td::Ref<vm::Cell> list);
   };
 
@@ -61,6 +63,10 @@ class SmartContract : public td::CntObject {
     td::uint64 amount{0};
     td::uint64 balance{0};
 
+    td::optional<block::StdAddress> address;
+    td::optional<std::shared_ptr<const block::Config>> config;
+    td::optional<vm::Dictionary> libraries;
+
     Args() {
     }
     Args(std::initializer_list<vm::StackEntry> stack)
@@ -106,6 +112,18 @@ class SmartContract : public td::CntObject {
       this->balance = balance;
       return std::move(*this);
     }
+    Args&& set_address(block::StdAddress address) {
+      this->address = address;
+      return std::move(*this);
+    }
+    Args&& set_config(std::shared_ptr<const block::Config>& config) {
+      this->config = config;
+      return std::move(*this);
+    }
+    Args&& set_libraries(vm::Dictionary libraries) {
+      this->libraries = libraries;
+      return std::move(*this);
+    }
 
     td::Result<td::int32> get_method_id() const {
       if (!method_id) {

crypto/test/test-bigint.cpp

@@ -0,0 +1,876 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <assert.h>
+#include <string.h>
+#include <unistd.h>
+#include <array>
+#include <string>
+#include <iostream>
+#include <sstream>
+#include <random>
+#include "common/refcnt.hpp"
+#include "common/bigint.hpp"
+#include "common/refint.h"
+#include "modbigint.cpp"
+
+#include "td/utils/tests.h"
+
+int mkint_chk_mode = -1, res_chk_mode = 0;
+long long iterations = 100000, cur_iteration = -1, debug_iteration = -2;
+#define IFDEBUG if (cur_iteration == debug_iteration || debug_iteration == -3)
+
+using BInt = modint::ModArray<18>;     // integers up to 2^537
+using MRInt = modint::MixedRadix<18>;  // auxiliary integer representation for printing, comparing etc
+
+MRInt p2_256, np2_256, p2_63, np2_63;
+constexpr long long ll_min = -2 * (1LL << 62), ll_max = ~ll_min;
+constexpr double dbl_pow256 = 1.1579208923731619542e77 /* 0x1p256 */;  // 2^256
+
+std::mt19937_64 Random(666);
+
+template <typename T>
+bool equal(td::RefInt256 x, T y) {
+  return !td::cmp(x, y);
+}
+
+bool equal_or_nan(td::RefInt256 x, td::RefInt256 y) {
+  return equal(x, y) || (!x->is_valid() && !y->fits_bits(257)) || (!y->is_valid() && !x->fits_bits(257));
+}
+
+#define CHECK_EQ(__x, __y) CHECK(equal(__x, __y))
+#define CHECK_EQ_NAN(__x, __y) CHECK(equal_or_nan(__x, __y))
+
+bool mr_in_range(const MRInt& x) {
+  return x < p2_256 && x >= np2_256;
+}
+
+bool mr_is_small(const MRInt& x) {
+  return x < p2_63 && x >= np2_63;
+}
+
+bool mr_fits_bits(const MRInt& x, int bits) {
+  if (bits > 0) {
+    return x < MRInt::pow2(bits - 1) && x >= MRInt::negpow2(bits - 1);
+  } else {
+    return !bits && !x.sgn();
+  }
+}
+
+bool mr_ufits_bits(const MRInt& x, int bits) {
+  return bits >= 0 && x.sgn() >= 0 && x < MRInt::pow2(bits);
+}
+
+struct ShowBin {
+  unsigned char* data;
+  ShowBin(unsigned char _data[64]) : data(_data) {
+  }
+};
+
+std::ostream& operator<<(std::ostream& os, ShowBin bin) {
+  int i = 0, s = bin.data[0];
+  if (s == 0 || s == 0xff) {
+    while (i < 64 && bin.data[i] == s) {
+      i++;
+    }
+  }
+  if (i >= 3) {
+    os << (s ? "ff..ff" : "00..00");
+  } else {
+    i = 0;
+  }
+  constexpr static char hex_digits[] = "0123456789abcdef";
+  while (i < 64) {
+    int t = bin.data[i++];
+    os << hex_digits[t >> 4] << hex_digits[t & 15];
+  }
+  return os;
+}
+
+std::ostream& operator<<(std::ostream& os, const td::AnyIntView<td::BigIntInfo>& x) {
+  os << '[';
+  for (int i = 0; i < x.size(); i++) {
+    os << ' ' << x.digits[i];
+  }
+  os << " ]";
+  return os;
+}
+
+template <typename T>
+bool extract_value_any_bool(BInt& val, const td::AnyIntView<T>& x, bool chk_norm = true) {
+  int n = x.size();
+  if (n <= 0 || n > x.max_size() || (!x.digits[n - 1] && n > 1)) {
+    return false;
+  }
+  assert(n == 1 || x.digits[n - 1] != 0);
+  val.set_zero();
+  for (int i = n - 1; i >= 0; --i) {
+    val.lshift_add(T::word_shift, x.digits[i]);
+    if (chk_norm && (x.digits[i] < -T::Half || x.digits[i] >= T::Half)) {
+      return false;  // unnormalized
+    }
+  }
+  return true;
+}
+
+template <typename T>
+bool extract_value_bool(BInt& val, const T& x, bool chk_norm = true) {
+  return extract_value_any_bool(val, x.as_any_int(), chk_norm);
+}
+
+BInt extract_value_any(const td::AnyIntView<td::BigIntInfo>& x, bool chk_norm = true) {
+  BInt res;
+  CHECK(extract_value_any_bool(res, x, chk_norm));
+  return res;
+}
+
+template <typename T>
+BInt extract_value(const T& x, bool chk_norm = true) {
+  return extract_value_any(x.as_any_int(), chk_norm);
+}
+
+template <typename T>
+BInt extract_value_alt(const T& x) {
+  BInt res;
+  const int* md = res.mod_array();
+  for (int i = 0; i < res.n / 2; i++) {
+    T copy{x};
+    int m1 = md[2 * i], m2 = md[2 * i + 1];
+    long long rem = copy.divmod_short((long long)m1 * m2);
+    res.a[2 * i] = (int)(rem % m1);
+    res.a[2 * i + 1] = (int)(rem % m2);
+  }
+  if (res.n & 1) {
+    T copy{x};
+    res.a[res.n - 1] = (int)copy.divmod_short(md[res.n - 1]);
+  }
+  return res;
+}
+
+constexpr int min_spec_int = -0xfd08, max_spec_int = 0xfd07;
+// x = sgn*(ord*256+a*16+b) => sgn*((32+a)*2^(ord-2) + b - 8)
+// x = -0xfd08 => -2^256 ... x = 0xfd07 => 2^256 - 1
+td::RefInt256 make_special_int(int x, BInt* ptr = nullptr, unsigned char bin[64] = nullptr) {
+  bool sgn = (x < 0);
+  if (sgn) {
+    x = -x;
+  }
+  int ord = (x >> 8) - 2, a = 32 + ((x >> 4) & 15), b = (x & 15) - 8;
+  if (ord < 0) {
+    a >>= -ord;
+    ord = 0;
+  }
+  if (sgn) {
+    a = -a;
+    b = -b;
+  }
+  if (ptr) {
+    ptr->set_int(a);
+    *ptr <<= ord;
+    *ptr += b;
+  }
+  if (bin) {
+    int acc = b, r = ord;
+    for (int i = 63; i >= 0; --i) {
+      if (r < 8) {
+        acc += (a << r);
+        r = 1024;
+      }
+      r -= 8;
+      bin[i] = (unsigned char)(acc & 0xff);
+      acc >>= 8;
+    }
+  }
+  return (td::make_refint(a) << ord) + b;
+}
+
+int rand_int(int min, int max) {
+  return min + (int)(Random() % (max - min + 1));
+}
+
+unsigned randu() {
+  return (unsigned)(Random() << 16);
+}
+
+bool coin() {
+  return Random() & (1 << 28);
+}
+
+// returns 0 with probability 1/2, 1 with prob. 1/4, ..., k with prob. 1/2^(k+1)
+int randexp(int max = 63, int min = 0) {
+  return min + __builtin_clzll(Random() | (1ULL << (63 - max + min)));
+}
+
+void bin_add_small(unsigned char bin[64], long long val, int shift = 0) {
+  val <<= shift & 7;
+  for (int i = 63 - (shift >> 3); i >= 0 && val; --i) {
+    val += bin[i];
+    bin[i] = (unsigned char)val;
+    val >>= 8;
+  }
+}
+
+// adds sgn * (random number less than 2^(ord - ord2)) * 2^ord2
+td::RefInt256 add_random_bits(td::RefInt256 x, BInt& val, unsigned char bin[64], int ord2, int ord, int sgn = 1) {
+  int t;
+  do {
+    t = std::max((ord - 1) & -16, ord2);
+    int a = sgn * rand_int(0, (1 << (ord - t)) - 1);
+    // add a << t
+    val.add_lshift(t, a);
+    x += td::make_refint(a) << t;
+    bin_add_small(bin, a, t);
+    ord = t;
+  } while (t > ord2);
+  return x;
+}
+
+// generates a random integer in range -2^256 .. 2^256-1 (and sometimes outside)
+// distribution is skewed towards +/- 2^n +/- 2^n +/- smallint, but completely random integers are also generated
+td::RefInt256 make_random_int0(BInt& val, unsigned char bin[64]) {
+  memset(bin, 0, 64);
+  int ord = rand_int(-257, 257);
+  if (ord <= 2 && ord >= -2) {
+    // -2..2 represent themselves
+    val.set_int(ord);
+    bin_add_small(bin, ord);
+    return td::make_refint(ord);
+  }
+  int sgn = (ord < 0 ? -1 : 1);
+  ord = sgn * ord - 1;
+  int f = std::min(ord, randexp(15)), a = sgn * rand_int(1 << f, (2 << f) - 1);
+  ord -= f;
+  // first summand is a << ord
+  auto res = td::make_refint(a) << ord;
+  val.set_int(a);
+  val <<= ord;
+  bin_add_small(bin, a, ord);
+  if (!ord) {
+    // all bits ready
+    return res;
+  }
+  for (int s = 0; s < 2 && ord; s++) {
+    // decide whether we want an intermediate order (50%), and whether we want randomness above/below that order
+    int ord2 = (s ? 0 : std::max(0, rand_int(~ord, ord - 1)));
+    if (!rand_int(0, 4)) {  // 20%
+      // random bits between ord2 and ord
+      res = add_random_bits(std::move(res), val, bin, ord2, ord, sgn);
+    }
+    if (rand_int(0, 4)) {  // 80%
+      // non-zero adjustment
+      f = randexp(15);
+      a = rand_int(-(2 << f) + 1, (2 << f) - 1);
+      ord = std::max(ord2 - f, 0);
+      // add a << ord
+      val.add_lshift(ord, a);
+      res += (td::make_refint(a) << ord);
+      bin_add_small(bin, a, ord);
+    }
+  }
+  return res;
+}
+
+td::RefInt256 make_random_int(BInt& val, unsigned char bin[64]) {
+  while (true) {
+    auto res = make_random_int0(val, bin);
+    if (res->fits_bits(257)) {
+      return res;
+    }
+  }
+}
+
+void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt* valptr = nullptr,
+                        const unsigned char bin[64] = nullptr) {
+  CHECK(x.not_null() && (in_range <= -2 || x->is_valid()));
+  if (!x->is_valid()) {
+    // not much to check when x is a NaN
+    unsigned char bytes[64];
+    if (valptr) {
+      // check that the true answer at `valptr` is out of range
+      CHECK(!mr_in_range(valptr->to_mixed_radix()));
+      if (mode & 0x200) {
+        // check BInt binary export
+        valptr->to_binary(bytes, 64);
+        if (bin) {
+          // check that the two true answers match
+          CHECK(!memcmp(bin, bytes, 64));
+        } else {
+          bin = bytes;
+        }
+      }
+    }
+    if (bin) {
+      // check that the true answer in `bin` is out of range
+      int i = 0, sgn = (bin[0] >= 0x80 ? -1 : 0);
+      while (i < 32 && bin[i] == (unsigned char)sgn)
+        ;
+      CHECK(i < 32);
+      if (valptr && (mode & 0x100)) {
+        // check BInt binary export
+        BInt val2;
+        val2.from_binary(bin, 64);
+        CHECK(*valptr == val2);
+      }
+    }
+    return;
+  }
+  unsigned char bytes[64];
+  CHECK(x->export_bytes(bytes, 64));
+  if (bin) {
+    CHECK(!memcmp(bytes, bin, 64));
+  }
+  BInt val = extract_value(*x);
+  if (valptr) {
+    if (val != *valptr) {
+      std::cerr << "extracted " << val << " from " << x << ' ' << x->as_any_int() << ", expected " << *valptr
+                << std::endl;
+    }
+    CHECK(val == *valptr);
+  }
+  if (mode & 1) {
+    BInt val2 = extract_value_alt(*x);
+    CHECK(val == val2);
+  }
+  if (mode & 2) {
+    // check binary import
+    td::BigInt256 y;
+    y.import_bytes(bytes, 64);
+    CHECK(y == *x);
+  }
+  if (mode & 0x100) {
+    // check binary import for BInt
+    BInt val2;
+    val2.from_binary(bytes, 64);
+    CHECK(val == val2);
+  }
+  // check if small (fits into 64 bits)
+  long long xval = (long long)val;
+  bool is_small = (xval != ll_min || val == xval);
+  CHECK(is_small == x->fits_bits(64));
+  if (is_small) {
+    // special check for small (64-bit) values
+    CHECK(x->to_long() == xval);
+    CHECK((long long)__builtin_bswap64(*(long long*)(bytes + 64 - 8)) == xval);
+    CHECK(in_range);
+    // check sign
+    CHECK(x->sgn() == (xval > 0 ? 1 : (xval < 0 ? -1 : 0)));
+    // check comparison with long long
+    CHECK(x == xval);
+    CHECK(!cmp(x, xval));
+    if (mode & 4) {
+      // check constructor from long long
+      CHECK(!cmp(x, td::make_refint(xval)));
+      if (xval != ll_min) {
+        CHECK(x > xval - 1);
+        CHECK(x > td::make_refint(xval - 1));
+      }
+      if (xval != ll_max) {
+        CHECK(x < xval + 1);
+        CHECK(x < td::make_refint(xval + 1));
+      }
+    }
+    if (!(mode & ~0x107)) {
+      return;  // fast check for small ints in this case
+    }
+  }
+
+  MRInt mval(val);  // somewhat slow
+  bool val_in_range = mr_in_range(mval);
+  CHECK(x->fits_bits(257) == val_in_range);
+  if (in_range >= 0) {
+    CHECK((int)val_in_range == in_range);
+  }
+  if (mode & 0x200) {
+    // check binary export for BInt
+    unsigned char bytes2[64];
+    mval.to_binary(bytes2, 64);
+    CHECK(!memcmp(bytes, bytes2, 64));
+  }
+  // check sign
+  int sgn = mval.sgn();
+  CHECK(x->sgn() == sgn);
+  CHECK(is_small == mr_is_small(mval));
+  if (is_small) {
+    CHECK((long long)mval == xval);
+  }
+  if (mode & 0x10) {
+    // check decimal export
+    std::string dec = mval.to_dec_string();
+    CHECK(x->to_dec_string() == dec);
+    // check decimal import
+    td::BigInt256 y;
+    int l = y.parse_dec(dec);
+    CHECK((std::size_t)l == dec.size() && y == *x);
+    if (mode & 0x1000) {
+      // check decimal import for BInt
+      BInt val2;
+      CHECK(val2.from_dec_string(dec) && val2 == val);
+    }
+  }
+  if (mode & 0x20) {
+    // check binary bit size
+    int sz = x->bit_size();
+    CHECK(sz >= 0 && sz <= 300);
+    CHECK(x->fits_bits(sz) && (!sz || !x->fits_bits(sz - 1)));
+    CHECK(mr_fits_bits(mval, sz) && !mr_fits_bits(mval, sz - 1));
+    int usz = x->bit_size(false);
+    CHECK(sgn >= 0 || usz == 0x7fffffff);
+    if (sgn >= 0) {
+      CHECK(x->unsigned_fits_bits(usz) && (!usz || !x->unsigned_fits_bits(usz - 1)));
+      CHECK(mr_ufits_bits(mval, usz) && !mr_ufits_bits(mval, usz - 1));
+    } else {
+      CHECK(!x->unsigned_fits_bits(256) && !x->unsigned_fits_bits(300));
+    }
+  }
+}
+
+void init_aux() {
+  np2_256 = p2_256 = MRInt::pow2(256);
+  np2_256.negate();
+  CHECK(np2_256 == MRInt::negpow2(256));
+  p2_63 = np2_63 = MRInt::pow2(63);
+  np2_63.negate();
+  CHECK(np2_63 == MRInt::negpow2(63));
+}
+
+std::vector<td::RefInt256> SpecInt;
+BInt SpecIntB[max_spec_int - min_spec_int + 1];
+
+void init_check_special_ints() {
+  std::cerr << "check special ints" << std::endl;
+  BInt b;
+  unsigned char binary[64];
+  for (int idx = min_spec_int - 512; idx <= max_spec_int + 512; idx++) {
+    td::RefInt256 x = make_special_int(idx, &b, binary);
+    check_one_int_repr(x, mkint_chk_mode, idx >= min_spec_int && idx <= max_spec_int, &b, binary);
+    if (idx >= min_spec_int && idx <= max_spec_int) {
+      SpecIntB[idx - min_spec_int] = b;
+      SpecInt.push_back(std::move(x));
+    }
+  }
+}
+
+void check_res(td::RefInt256 y, const BInt& yv) {
+  check_one_int_repr(std::move(y), res_chk_mode, -2, &yv);
+}
+
+void check_unary_ops_on(td::RefInt256 x, const BInt& xv) {
+  // NEGATE
+  BInt yv = -xv;
+  check_res(-x, yv);
+  // NOT
+  check_res(~x, yv -= 1);
+}
+
+void check_unary_ops() {
+  std::cerr << "check unary ops" << std::endl;
+  for (int idx = min_spec_int; idx <= max_spec_int; idx++) {
+    check_unary_ops_on(SpecInt[idx - min_spec_int], SpecIntB[idx - min_spec_int]);
+  }
+}
+
+void check_pow2_ops(int shift) {
+  // POW2
+  td::RefInt256 r{true};
+  r.unique_write().set_pow2(shift);
+  check_res(r, BInt::pow2(shift));
+  // POW2DEC
+  r.unique_write().set_pow2(shift).add_tiny(-1).normalize();
+  check_res(r, BInt::pow2(shift) - 1);
+  // NEGPOW2
+  r.unique_write().set_pow2(shift).negate().normalize();
+  check_res(r, -BInt::pow2(shift));
+}
+
+void check_pow2_ops() {
+  std::cerr << "check power-2 ops" << std::endl;
+  for (int i = 0; i <= 256; i++) {
+    check_pow2_ops(i);
+  }
+}
+
+void check_shift_ops_on(int shift, td::RefInt256 x, const BInt& xv, const MRInt& mval) {
+  // LSHIFT
+  check_res(x << shift, xv << shift);
+  // FITS
+  CHECK(x->fits_bits(shift) == mr_fits_bits(mval, shift));
+  // UFITS
+  CHECK(x->unsigned_fits_bits(shift) == mr_ufits_bits(mval, shift));
+  // ADDPOW2 / SUBPOW2
+  auto y = x;
+  y.write().add_pow2(shift).normalize();
+  check_res(std::move(y), xv + BInt::pow2(shift));
+  y = x;
+  y.write().sub_pow2(shift).normalize();
+  check_res(std::move(y), xv - BInt::pow2(shift));
+  // RSHIFT, MODPOW2
+  for (int round_mode = -1; round_mode <= 1; round_mode++) {
+    auto r = x, q = td::rshift(x, shift, round_mode);  // RSHIFT
+    CHECK(q.not_null() && q->is_valid());
+    r.write().mod_pow2(shift, round_mode).normalize();  // MODPOW2
+    CHECK(r.not_null() && r->is_valid());
+    if (round_mode < 0) {
+      CHECK(!cmp(x >> shift, q));  // operator>> should be equivalent to td::rshift
+    }
+    BInt qv = extract_value(*q), rv = extract_value(*r);
+    // check main division equality (q << shift) + r == x
+    CHECK((qv << shift) + rv == xv);
+    MRInt rval(rv);
+    // check remainder range
+    switch (round_mode) {
+      case 1:
+        rval.negate();  // fallthrough
+      case -1:
+        CHECK(mr_ufits_bits(rval, shift));
+        break;
+      case 0:
+        CHECK(mr_fits_bits(rval, shift));
+    }
+  }
+}
+
+void check_shift_ops() {
+  std::cerr << "check left/right shift ops" << std::endl;
+  for (int idx = min_spec_int; idx <= max_spec_int; idx++) {
+    //for (int idx : {-52240, -52239, -52238, -3, -2, -1, 0, 1, 2, 3, 52238, 52239, 52240}) {
+    const auto& xv = SpecIntB[idx - min_spec_int];
+    MRInt mval(xv);
+    if (!(idx % 1000)) {
+      std::cerr << "# " << idx << " : " << mval << std::endl;
+    }
+    for (int i = 0; i <= 256; i++) {
+      check_shift_ops_on(i, SpecInt[idx - min_spec_int], xv, mval);
+    }
+  }
+}
+
+void check_remainder_range(BInt& rv, const BInt& dv, int rmode = -1) {
+  if (rmode > 0) {
+    rv.negate();
+  } else if (!rmode) {
+    rv *= 2;
+  }
+  MRInt d(dv), r(rv);
+  int ds = d.sgn(), rs = r.sgn();
+  //std::cerr << "rmode=" << rmode << " ds=" << ds << " rs=" << rs << " d=" << d << " r=" << r << std::endl;
+  if (!rs) {
+    return;
+  }
+  if (rmode) {
+    // must have 0 < r < d or 0 > r > d
+    //if (rs != ds) std::cerr << "iter=" << cur_iteration << " : rmode=" << rmode << " ds=" << ds << " rs=" << rs << " d=" << d << " r=" << r << std::endl;
+    CHECK(rs == ds);
+    CHECK(ds * r.cmp(d) < 0);
+  } else {
+    // must have -d <= r < d or -d >= r > d
+    if (rs == -ds) {
+      r.negate();
+      CHECK(ds * r.cmp(d) <= 0);
+    } else {
+      CHECK(ds * r.cmp(d) < 0);
+    }
+  }
+}
+
+void check_divmod(td::RefInt256 x, const BInt& xv, long long xl, td::RefInt256 y, const BInt& yv, long long yl,
+                  int rmode = -2) {
+  if (rmode < -1) {
+    //IFDEBUG std::cerr << "  divide " << x << " / " << y << std::endl;
+    for (rmode = -1; rmode <= 1; rmode++) {
+      check_divmod(x, xv, xl, y, yv, yl, rmode);
+    }
+    return;
+  }
+  auto dm = td::divmod(x, y, rmode);
+  auto q = std::move(dm.first), r = std::move(dm.second);
+  if (!yl) {
+    // division by zero
+    CHECK(q.not_null() && !q->is_valid() && r.not_null() && !r->is_valid());
+    return;
+  }
+  CHECK(q.not_null() && q->is_valid() && r.not_null() && r->is_valid());
+  CHECK_EQ(x, y * q + r);
+  BInt qv = extract_value(*q), rv = extract_value(*r);
+  CHECK(xv == yv * qv + rv);
+  //IFDEBUG std::cerr << "    quot=" << q << " rem=" << r << std::endl;
+  check_remainder_range(rv, yv, rmode);
+  if (yl != ll_min && rmode == -1) {
+    // check divmod_short()
+    auto qq = x;
+    auto rem = qq.write().divmod_short(yl);
+    qq.write().normalize();
+    CHECK(qq->is_valid());
+    CHECK_EQ(qq, q);
+    CHECK(r == rem);
+    if (xl != ll_min) {
+      auto dm = std::lldiv(xl, yl);
+      if (dm.rem && (dm.rem ^ yl) < 0) {
+        dm.rem += yl;
+        dm.quot--;
+      }
+      CHECK(q == dm.quot);
+      CHECK(r == dm.rem);
+    }
+  }
+}
+
+void check_binary_ops_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv) {
+  bool x_small = x->fits_bits(62), y_small = y->fits_bits(62);  // not 63
+  long long xl = x_small ? x->to_long() : ll_min, yl = y_small ? y->to_long() : ll_min;
+  if (x_small) {
+    CHECK(x == xl);
+  }
+  if (y_small) {
+    CHECK(y == yl);
+  }
+  // ADD, ADDR
+  auto z = x + y, w = y + x;
+  CHECK_EQ(z, w);
+  check_res(z, xv + yv);
+  // ADDCONST
+  if (y_small) {
+    CHECK_EQ(z, x + yl);
+  }
+  if (x_small) {
+    CHECK_EQ(z, y + xl);
+  }
+  if (x_small && y_small) {
+    CHECK_EQ(z, xl + yl);
+  }
+  // SUB
+  z = x - y;
+  check_res(z, xv - yv);
+  // SUBCONST
+  if (y_small) {
+    CHECK_EQ(z, x - yl);
+    if (x_small) {
+      CHECK_EQ(z, xl - yl);
+    }
+  }
+  // SUBR
+  z = y - x;
+  check_res(z, yv - xv);
+  if (x_small) {
+    CHECK_EQ(z, y - xl);
+    if (y_small) {
+      CHECK_EQ(z, yl - xl);
+    }
+  }
+  // CMP
+  MRInt xmr(xv), ymr(yv);
+  int cmpv = xmr.cmp(ymr);
+  CHECK(td::cmp(x, y) == cmpv);
+  CHECK(td::cmp(y, x) == -cmpv);
+  if (y_small) {
+    CHECK(td::cmp(x, yl) == cmpv);
+  }
+  if (x_small) {
+    CHECK(td::cmp(y, xl) == -cmpv);
+  }
+  if (x_small && y_small) {
+    CHECK(cmpv == (xl < yl ? -1 : (xl > yl ? 1 : 0)));
+  }
+  // MUL
+  z = x * y;
+  BInt zv = xv * yv;
+  check_res(z, zv);
+  CHECK_EQ(z, y * x);
+  // MULCONST
+  if (y_small) {
+    CHECK_EQ_NAN(z, x * yl);
+  }
+  if (x_small) {
+    CHECK_EQ_NAN(z, y * xl);
+  }
+  if (x_small && y_small && (!yl || std::abs(xl) <= ll_max / std::abs(yl))) {
+    CHECK_EQ(z, xl * yl);
+  }
+  // DIVMOD
+  if (z->fits_bits(257)) {
+    int adj = 2 * rand_int(-2, 2) - (int)z->is_odd();
+    z += adj;
+    z >>= 1;
+    zv += adj;
+    zv >>= 1;
+    // z is approximately x * y / 2; divide by y
+    check_divmod(z, zv, z->fits_bits(62) ? z->to_long() : ll_min, y, yv, yl);
+  }
+  check_divmod(x, xv, xl, y, yv, yl);
+}
+
+void finish_check_muldivmod(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, td::RefInt256 z,
+                            const BInt& zv, td::RefInt256 q, td::RefInt256 r, int rmode) {
+  static constexpr double eps = 1e-14;
+  CHECK(q.not_null() && r.not_null());
+  //std::cerr << " muldivmod: " << xv << " * " << yv << " / " << zv << " (round " << rmode << ") = " << q << " " << r << std::endl;
+  if (!zv) {
+    // division by zero
+    CHECK(!q->is_valid() && !r->is_valid());
+    return;
+  }
+  CHECK(r->is_valid());  // remainder always exists if y != 0
+  BInt xyv = xv * yv, rv = extract_value(*r);
+  MRInt xy_mr(xyv), z_mr(zv);
+  double q0 = (double)xy_mr / (double)z_mr;
+  if (std::abs(q0) < 1.01 * dbl_pow256) {
+    // result more or less in range
+    CHECK(q->is_valid());
+  } else if (!q->is_valid()) {
+    // result out of range, NaN is an acceptable answer
+    // check that x * y - r is divisible by z
+    xyv -= rv;
+    xyv /= zv;
+    xy_mr = xyv;
+    double q1 = (double)xy_mr;
+    CHECK(std::abs(q1 - q0) < eps * std::abs(q0));
+  } else {
+    BInt qv = extract_value(*q);
+    // must have x * y = z * q + r
+    CHECK(xv * yv == zv * qv + rv);
+  }
+  // check that r is in correct range [0, z) or [0, -z) or [-z/2, z/2)
+  check_remainder_range(rv, zv, rmode);
+}
+
+void check_muldivmod_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, td::RefInt256 z,
+                        const BInt& zv, int rmode = 2) {
+  if (rmode < -1) {
+    for (rmode = -1; rmode <= 1; rmode++) {
+      check_muldivmod_on(x, xv, y, yv, z, zv, rmode);
+    }
+    return;
+  } else if (rmode > 1) {
+    rmode = rand_int(-1, 1);
+  }
+  // MULDIVMOD
+  auto qr = td::muldivmod(x, y, z, rmode);
+  finish_check_muldivmod(std::move(x), xv, std::move(y), yv, std::move(z), zv, std::move(qr.first),
+                         std::move(qr.second), rmode);
+}
+
+void check_mul_rshift_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, int shift, int rmode = 2) {
+  if (rmode < -1) {
+    for (rmode = -1; rmode <= 1; rmode++) {
+      check_mul_rshift_on(x, xv, y, yv, shift, rmode);
+    }
+    return;
+  } else if (rmode > 1) {
+    rmode = rand_int(-1, 1);
+  }
+  // MULRSHIFTMOD
+  typename td::BigInt256::DoubleInt tmp{0};
+  tmp.add_mul(*x, *y);
+  typename td::BigInt256::DoubleInt tmp2{tmp};
+  tmp2.rshift(shift, rmode).normalize();
+  tmp.normalize().mod_pow2(shift, rmode).normalize();
+  finish_check_muldivmod(std::move(x), xv, std::move(y), yv, {}, BInt::pow2(shift), td::make_refint(tmp2),
+                         td::make_refint(tmp), rmode);
+}
+
+void check_lshift_div_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, int shift, int rmode = 2) {
+  if (rmode < -1) {
+    for (rmode = -1; rmode <= 1; rmode++) {
+      check_lshift_div_on(x, xv, y, yv, shift, rmode);
+    }
+    return;
+  } else if (rmode > 1) {
+    rmode = rand_int(-1, 1);
+  }
+  // LSHIFTDIV
+  typename td::BigInt256::DoubleInt tmp{*x}, quot;
+  tmp <<= shift;
+  tmp.mod_div(*y, quot, rmode);
+  quot.normalize();
+  finish_check_muldivmod(std::move(x), xv, {}, BInt::pow2(shift), std::move(y), yv, td::make_refint(quot),
+                         td::make_refint(tmp), rmode);
+}
+
+void check_random_ops() {
+  constexpr long long chk_it = 100000;
+  std::cerr << "check random ops (" << iterations << " iterations)" << std::endl;
+  BInt xv, yv, zv;
+  unsigned char xbin[64], ybin[64], zbin[64];
+  for (cur_iteration = 0; cur_iteration < iterations; cur_iteration++) {
+    auto x = make_random_int0(xv, xbin);
+    if (!(cur_iteration % 10000)) {
+      std::cerr << "#" << cur_iteration << ": check on " << xv << " = " << ShowBin(xbin) << " = " << x->as_any_int()
+                << std::endl;
+    }
+    check_one_int_repr(x, cur_iteration < chk_it ? -1 : 0, -1, &xv, xbin);
+    MRInt xmr(xv);
+    if (!x->fits_bits(257)) {
+      continue;
+    }
+    check_unary_ops_on(x, xv);
+    for (int j = 0; j < 10; j++) {
+      int shift = rand_int(0, 256);
+      //std::cerr << "check shift by " << shift << std::endl;
+      check_shift_ops_on(shift, x, xv, xmr);
+      auto y = make_random_int(yv, ybin);
+      //std::cerr << "  y = " << y << " = " << yv << " = " << ShowBin(ybin) << " = " << y->as_any_int() << std::endl;
+      check_one_int_repr(y, 0, 1, &yv, ybin);
+      check_binary_ops_on(x, xv, y, yv);
+      //std::cerr << "  *>> " << shift << std::endl;
+      check_mul_rshift_on(x, xv, y, yv, shift);
+      //std::cerr << "  <</ " << shift << std::endl;
+      check_lshift_div_on(x, xv, y, yv, shift);
+      auto z = make_random_int(zv, zbin);
+      //std::cerr << "  */ z = " << z << " = " << zv << " = " << ShowBin(zbin) << " = " << z->as_any_int() << std::endl;
+      check_muldivmod_on(x, xv, y, yv, z, zv);
+    }
+  }
+}
+
+void check_special() {
+  std::cerr << "run special tests" << std::endl;
+  check_divmod((td::make_refint(-1) << 207) - 1, BInt::negpow2(207) - 1, ll_min, (td::make_refint(1) << 207) - 1,
+               BInt::pow2(207) - 1, ll_min);
+}
+
+int main(int argc, char* const argv[]) {
+  bool do_check_shift_ops = false;
+  int i;
+  while ((i = getopt(argc, argv, "hSs:i:")) != -1) {
+    switch (i) {
+      case 'S':
+        do_check_shift_ops = true;
+        break;
+      case 's':
+        Random.seed(atoll(optarg));
+        break;
+      case 'i':
+        iterations = atoll(optarg);
+        break;
+      default:
+        std::cerr << "unknown option: " << (char)i << std::endl;
+        // fall through
+      case 'h':
+        std::cerr << "usage:\t" << argv[0] << " [-S] [-i<random-op-iterations>] [-s<random-seed>]" << std::endl;
+        return 2;
+    }
+  }
+  modint::init();
+  init_aux();
+  init_check_special_ints();
+  check_pow2_ops();
+  check_unary_ops();
+  if (do_check_shift_ops) {
+    check_shift_ops();
+  }
+  check_special();
+  check_random_ops();
+  return 0;
+}

crypto/vm/arithops.cpp

@@ -387,18 +387,16 @@ int exec_muldivmod(VmState* st, unsigned args, int quiet) {
   auto z = stack.pop_int();
   auto y = stack.pop_int();
   auto x = stack.pop_int();
-  typename td::BigInt256::DoubleInt tmp{0};
+  typename td::BigInt256::DoubleInt tmp{0}, quot;
   tmp.add_mul(*x, *y);
   auto q = td::make_refint();
-  tmp.mod_div(*z, q.unique_write(), round_mode);
+  tmp.mod_div(*z, quot, round_mode);
   switch ((args >> 2) & 3) {
     case 1:
-      q.unique_write().normalize();
+      stack.push_int_quiet(td::make_refint(quot.normalize()), quiet);
-      stack.push_int_quiet(std::move(q), quiet);
       break;
     case 3:
-      q.unique_write().normalize();
+      stack.push_int_quiet(td::make_refint(quot.normalize()), quiet);
-      stack.push_int_quiet(std::move(q), quiet);
       // fallthrough
     case 2:
       stack.push_int_quiet(td::make_refint(tmp), quiet);
@@ -459,7 +457,7 @@ int exec_mulshrmod(VmState* st, unsigned args, int mode) {
     }
       // fallthrough
     case 2:
-      tmp.mod_pow2(z, round_mode).normalize();
+      tmp.normalize().mod_pow2(z, round_mode).normalize();
       stack.push_int_quiet(td::make_refint(tmp), mode & 1);
       break;
   }
@@ -520,21 +518,17 @@ int exec_shldivmod(VmState* st, unsigned args, int mode) {
   }
   auto z = stack.pop_int();
   auto x = stack.pop_int();
-  typename td::BigInt256::DoubleInt tmp{*x};
+  typename td::BigInt256::DoubleInt tmp{*x}, quot;
   tmp <<= y;
   switch ((args >> 2) & 3) {
     case 1: {
-      auto q = td::make_refint();
+      tmp.mod_div(*z, quot, round_mode);
-      tmp.mod_div(*z, q.unique_write(), round_mode);
+      stack.push_int_quiet(td::make_refint(quot.normalize()), mode & 1);
-      q.unique_write().normalize();
-      stack.push_int_quiet(std::move(q), mode & 1);
       break;
     }
     case 3: {
-      auto q = td::make_refint();
+      tmp.mod_div(*z, quot, round_mode);
-      tmp.mod_div(*z, q.unique_write(), round_mode);
+      stack.push_int_quiet(td::make_refint(quot.normalize()), mode & 1);
-      q.unique_write().normalize();
-      stack.push_int_quiet(std::move(q), mode & 1);
       stack.push_int_quiet(td::make_refint(tmp), mode & 1);
       break;
     }

crypto/vm/boc.cpp

@@ -475,15 +475,128 @@ std::string BagOfCells::extract_string() const {
   return std::string{serialized.data(), serialized.data() + serialized.size()};
 }
 
-void BagOfCells::store_uint(unsigned long long value, unsigned bytes) {
+namespace {
+struct BufferWriter {
+  BufferWriter(unsigned char* store_start, unsigned char* store_end)
+      : store_start(store_start), store_ptr(store_start), store_end(store_end) {}
+
+  size_t position() const {
+    return store_ptr - store_start;
+  }
+  size_t remaining() const {
+    return store_end - store_ptr;
+  }
+  void chk() const {
+    DCHECK(store_ptr <= store_end);
+  }
+  bool empty() const {
+    return store_ptr == store_end;
+  }
+  void store_uint(unsigned long long value, unsigned bytes) {
     unsigned char* ptr = store_ptr += bytes;
-  store_chk();
+    chk();
     while (bytes) {
       *--ptr = value & 0xff;
       value >>= 8;
       --bytes;
     }
     DCHECK(!bytes);
+  }
+  void store_bytes(unsigned char const* data, size_t s) {
+    store_ptr += s;
+    chk();
+    memcpy(store_ptr - s, data, s);
+  }
+  unsigned get_crc32() const {
+    return td::crc32c(td::Slice{store_start, store_ptr});
+  }
+
+ private:
+  unsigned char* store_start;
+  unsigned char* store_ptr;
+  unsigned char* store_end;
+};
+
+struct FileWriter {
+  FileWriter(td::FileFd& fd, size_t expected_size)
+      : fd(fd), expected_size(expected_size) {}
+
+  ~FileWriter() {
+    flush();
+  }
+
+  size_t position() const {
+    return flushed_size + writer.position();
+  }
+  size_t remaining() const {
+    return expected_size - position();
+  }
+  void chk() const {
+    DCHECK(position() <= expected_size);
+  }
+  bool empty() const {
+    return remaining() == 0;
+  }
+  void store_uint(unsigned long long value, unsigned bytes) {
+    flush_if_needed(bytes);
+    writer.store_uint(value, bytes);
+  }
+  void store_bytes(unsigned char const* data, size_t s) {
+    flush_if_needed(s);
+    writer.store_bytes(data, s);
+  }
+  unsigned get_crc32() const {
+    unsigned char const* start = buf.data();
+    unsigned char const* end = start + writer.position();
+    return td::crc32c_extend(current_crc32, td::Slice(start, end));
+  }
+
+  td::Status finalize() {
+    flush();
+    return std::move(res);
+  }
+
+ private:
+  void flush_if_needed(size_t s) {
+    DCHECK(s <= BUF_SIZE);
+    if (s > BUF_SIZE - writer.position()) {
+      flush();
+    }
+  }
+
+  void flush() {
+    chk();
+    unsigned char* start = buf.data();
+    unsigned char* end = start + writer.position();
+    if (start == end) {
+      return;
+    }
+    flushed_size += end - start;
+    current_crc32 = td::crc32c_extend(current_crc32, td::Slice(start, end));
+    if (res.is_ok()) {
+      while (end > start) {
+        auto R = fd.write(td::Slice(start, end));
+        if (R.is_error()) {
+          res = R.move_as_error();
+          break;
+        }
+        size_t s = R.move_as_ok();
+        start += s;
+      }
+    }
+    writer = BufferWriter(buf.data(), buf.data() + buf.size());
+  }
+
+  td::FileFd& fd;
+  size_t expected_size;
+  size_t flushed_size = 0;
+  unsigned current_crc32 = td::crc32c(td::Slice());
+
+  static const size_t BUF_SIZE = 1 << 22;
+  std::vector<unsigned char> buf = std::vector<unsigned char>(BUF_SIZE, '\0');
+  BufferWriter writer = BufferWriter(buf.data(), buf.data() + buf.size());
+  td::Status res = td::Status::OK();
+};
 }
 
 //serialized_boc#672fb0ac has_idx:(## 1) has_crc32c:(## 1)
@@ -497,13 +610,16 @@ void BagOfCells::store_uint(unsigned long long value, unsigned bytes) {
 //  index:(cells * ##(off_bytes * 8))
 //  cell_data:(tot_cells_size * [ uint8 ])
 //  = BagOfCells;
-std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) {
+template<typename WriterT>
-  std::size_t size_est = estimate_serialized_size(mode);
+std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) {
-  if (!size_est || size_est > buff_size) {
+  auto store_ref = [&](unsigned long long value) {
-    return 0;
+    writer.store_uint(value, info.ref_byte_size);
-  }
+  };
-  init_store(buffer, buffer + size_est);
+  auto store_offset = [&](unsigned long long value) {
-  store_uint(info.magic, 4);
+    writer.store_uint(value, info.offset_byte_size);
+  };
+
+  writer.store_uint(info.magic, 4);
 
   td::uint8 byte{0};
   if (info.has_index) {
@@ -520,9 +636,9 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz
     return 0;
   }
   byte |= static_cast<td::uint8>(info.ref_byte_size);
-  store_uint(byte, 1);
+  writer.store_uint(byte, 1);
 
-  store_uint(info.offset_byte_size, 1);
+  writer.store_uint(info.offset_byte_size, 1);
   store_ref(cell_count);
   store_ref(root_count);
   store_ref(0);
@@ -532,7 +648,7 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz
     DCHECK(k >= 0 && k < cell_count);
     store_ref(k);
   }
-  DCHECK(store_ptr - buffer == (long long)info.index_offset);
+  DCHECK(writer.position() == info.index_offset);
   DCHECK((unsigned)cell_count == cell_list_.size());
   if (info.has_index) {
     std::size_t offs = 0;
@@ -551,8 +667,8 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz
     }
     DCHECK(offs == info.data_size);
   }
-  DCHECK(store_ptr - buffer == (long long)info.data_offset);
+  DCHECK(writer.position() == info.data_offset);
-  unsigned char* keep_ptr = store_ptr;
+  size_t keep_position = writer.position();
   for (int i = 0; i < cell_count; ++i) {
     const auto& dc_info = cell_list_[cell_count - 1 - i];
     const Ref<DataCell>& dc = dc_info.dc_ref;
@@ -560,9 +676,9 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz
     if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) {
       with_hash = true;
     }
-    int s = dc->serialize(store_ptr, 256, with_hash);
+    unsigned char buf[256];
-    store_ptr += s;
+    int s = dc->serialize(buf, 256, with_hash);
-    store_chk();
+    writer.store_bytes(buf, s);
     DCHECK(dc->size_refs() == dc_info.ref_num);
     // std::cerr << (dc_info.is_special() ? '*' : ' ') << i << '<' << (int)dc_info.wt << ">:";
     for (unsigned j = 0; j < dc_info.ref_num; ++j) {
@@ -573,16 +689,38 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz
     }
     // std::cerr << std::endl;
   }
-  store_chk();
+  writer.chk();
-  DCHECK(store_ptr - keep_ptr == (long long)info.data_size);
+  DCHECK(writer.position() - keep_position == info.data_size);
-  DCHECK(store_end - store_ptr == (info.has_crc32c ? 4 : 0));
+  DCHECK(writer.remaining() == (info.has_crc32c ? 4 : 0));
   if (info.has_crc32c) {
-    // compute crc32c of buffer .. store_ptr
+    unsigned crc = writer.get_crc32();
-    unsigned crc = td::crc32c(td::Slice{buffer, store_ptr});
+    writer.store_uint(td::bswap32(crc), 4);
-    store_uint(td::bswap32(crc), 4);
   }
-  DCHECK(store_empty());
+  DCHECK(writer.empty());
-  return store_ptr - buffer;
+  return writer.position();
+}
+
+std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) {
+  std::size_t size_est = estimate_serialized_size(mode);
+  if (!size_est || size_est > buff_size) {
+    return 0;
+  }
+  BufferWriter writer{buffer, buffer + size_est};
+  return serialize_to_impl(writer, mode);
+}
+
+td::Status BagOfCells::serialize_to_file(td::FileFd& fd, int mode) {
+  std::size_t size_est = estimate_serialized_size(mode);
+  if (!size_est) {
+    return td::Status::Error("no cells to serialize to this bag of cells");
+  }
+  FileWriter writer{fd, size_est};
+  size_t s = serialize_to_impl(writer, mode);
+  TRY_STATUS(writer.finalize());
+  if (s != size_est) {
+    return td::Status::Error("error while serializing a bag of cells: actual serialized size differs from estimated");
+  }
+  return td::Status::OK();
 }
 
 unsigned long long BagOfCells::Info::read_int(const unsigned char* ptr, unsigned bytes) {

crypto/vm/boc.h

@@ -23,6 +23,7 @@
 #include "td/utils/buffer.h"
 #include "td/utils/HashMap.h"
 #include "td/utils/HashSet.h"
+#include "td/utils/port/FileFd.h"
 
 namespace vm {
 using td::Ref;
@@ -216,8 +217,6 @@ class BagOfCells {
   int max_depth{1024};
   Info info;
   unsigned long long data_bytes{0};
-  unsigned char* store_ptr{nullptr};
-  unsigned char* store_end{nullptr};
   td::HashMap<Hash, int> cells;
   struct CellInfo {
     Ref<DataCell> dc_ref;
@@ -267,6 +266,9 @@ class BagOfCells {
   std::string serialize_to_string(int mode = 0);
   td::Result<td::BufferSlice> serialize_to_slice(int mode = 0);
   std::size_t serialize_to(unsigned char* buffer, std::size_t buff_size, int mode = 0);
+  td::Status serialize_to_file(td::FileFd& fd, int mode = 0);
+  template<typename WriterT>
+  std::size_t serialize_to_impl(WriterT& writer, int mode = 0);
   std::string extract_string() const;
 
   td::Result<long long> deserialize(const td::Slice& data, int max_roots = default_max_roots);
@@ -295,23 +297,6 @@ class BagOfCells {
     cell_list_.clear();
   }
   td::uint64 compute_sizes(int mode, int& r_size, int& o_size);
-  void init_store(unsigned char* from, unsigned char* to) {
-    store_ptr = from;
-    store_end = to;
-  }
-  void store_chk() const {
-    DCHECK(store_ptr <= store_end);
-  }
-  bool store_empty() const {
-    return store_ptr == store_end;
-  }
-  void store_uint(unsigned long long value, unsigned bytes);
-  void store_ref(unsigned long long value) {
-    store_uint(value, info.ref_byte_size);
-  }
-  void store_offset(unsigned long long value) {
-    store_uint(value, info.offset_byte_size);
-  }
   void reorder_cells();
   int revisit(int cell_idx, int force = 0);
   unsigned long long get_idx_entry_raw(int index);

crypto/vm/vm.cpp

@@ -604,6 +604,7 @@ Ref<Cell> VmState::load_library(td::ConstBitPtr hash) {
       return lib;
     }
   }
+  missing_library = hash;
   return {};
 }
 

crypto/vm/vm.h

@@ -96,6 +96,7 @@ class VmState final : public VmStateInterface {
   td::int64 loaded_cells_count{0};
   int stack_trace{0}, debug_off{0};
   bool chksig_always_succeed{false};
+  td::ConstBitPtr missing_library{0};
 
  public:
   enum {
@@ -321,6 +322,9 @@ class VmState final : public VmStateInterface {
   Ref<OrdCont> ref_to_cont(Ref<Cell> cell) const {
     return td::make_ref<OrdCont>(load_cell_slice_ref(std::move(cell)), get_cp());
   }
+  td::ConstBitPtr get_missing_library() const {
+    return missing_library;
+  }
 
  private:
   void init_cregs(bool same_c3 = false, bool push_0 = true);

doc/tvm.tex

@@ -1531,6 +1531,7 @@ Examples:
 \item {\tt A934$tt$} --- same as {\tt RSHIFT $tt+1$}: ($x$ -- $\lfloor x\cdot 2^{-tt-1}\rfloor$).
 \item {\tt A938$tt$} --- {\tt MODPOW2 $tt+1$}: ($x$ -- $x\bmod 2^{tt+1}$).
 \item {\tt A985} --- {\tt MULDIVR} ($x$ $y$ $z$ -- $q'$), where $q'=\lfloor xy/z+1/2\rfloor$.
+\item {\tt A988} --- {\tt MULMOD} ($x$ $y$ $z$ -- $r$), where $r=xy\bmod z=xy-qz$, $q=\lfloor xy/z\rfloor$. This operation always succeeds for $z\neq0$ and returns the correct value of~$r$, even if the intermediate result $xy$ or the quotient $q$ do not fit into 257 bits.
 \item {\tt A98C} --- {\tt MULDIVMOD} ($x$ $y$ $z$ -- $q$ $r$), where $q:=\lfloor x\cdot y/z\rfloor$, $r:=x\cdot y\bmod z$ (same as {\tt */MOD} in Forth).
 \item {\tt A9A4} --- {\tt MULRSHIFT} ($x$ $y$ $z$ -- $\lfloor xy\cdot2^{-z}\rfloor$) for $0\leq z\leq 256$.
 \item {\tt A9A5} --- {\tt MULRSHIFTR} ($x$ $y$ $z$ -- $\lfloor xy\cdot2^{-z}+1/2\rfloor$) for $0\leq z\leq 256$.
@@ -1576,10 +1577,12 @@ We opted to make all arithmetic operations ``non-quiet'' (signaling) by default,
 \begin{itemize}
 \item {\tt B7xx} --- {\tt QUIET} prefix, transforming any arithmetic operation into its ``quiet'' variant, indicated by prefixing a {\tt Q} to its mnemonic. Such operations return {\tt NaN}s instead of throwing integer overflow exceptions if the results do not fit in {\it Integer\/}s, or if one of their arguments is a {\tt NaN}. Notice that this does not extend to shift amounts and other parameters that must be within a small range (e.g., 0--1023). Also notice that this does not disable type-checking exceptions if a value of a type other than {\it Integer\/} is supplied. 
 \item {\tt B7A0} --- {\tt QADD} ($x$ $y$ -- $x+y$), always works if $x$ and $y$ are {\it Integer\/}s, but returns a {\tt NaN} if the addition cannot be performed.
+\item {\tt B7A8} --- {\tt QMUL} ($x$ $y$ -- $xy$), returns the product of $x$ and $y$ if $-2^{256}\leq xy<2^{256}$. Otherwise returns a {\tt NaN}, even if $x=0$ and $y$ is a {\tt NaN}.
 \item {\tt B7A904} --- {\tt QDIV} ($x$ $y$ -- $\lfloor x/y\rfloor$), returns a {\tt NaN} if $y=0$, or if $y=-1$ and $x=-2^{256}$, or if either of $x$ or $y$ is a {\tt NaN}.
+\item {\tt B7A98C} --- {\tt QMULDIVMOD} ($x$ $y$ $z$ -- $q$ $r$), where $q:=\lfloor x\cdot y/z\rfloor$, $r:=x\cdot y\bmod z$. If $z=0$, or if at least one of $x$, $y$, or $z$ is a {\tt NaN}, both $q$ and $r$ are set to {\tt NaN}. Otherwise the correct value of $r$ is always returned, but $q$ is replaced with {\tt NaN} if $q<-2^{256}$ or $q\geq2^{256}$.
 \item {\tt B7B0} --- {\tt QAND} ($x$ $y$ -- $x\&y$), bitwise ``and'' (similar to {\tt AND}), but returns a {\tt NaN} if either $x$ or $y$ is a {\tt NaN} instead of throwing an integer overflow exception. However, if one of the arguments is zero, and the other is a {\tt NaN}, the result is zero.
 \item {\tt B7B1} --- {\tt QOR} ($x$ $y$ -- $x\vee y$), bitwise ``or''. If $x=-1$ or $y=-1$, the result is always $-1$, even if the other argument is a {\tt NaN}.
-\item {\tt B7B507} --- {\tt QUFITS 8} ($x$ -- $x'$), checks whether $x$ is an unsigned byte (i.e., whether $0\leq x<2^8$), and replaces $x$ with a {\tt NaN} if this is not the case; leaves $x$ intact otherwise (i.e., if $x$ is an unsigned byte).
+\item {\tt B7B507} --- {\tt QUFITS 8} ($x$ -- $x'$), checks whether $x$ is an unsigned byte (i.e., whether $0\leq x<2^8$), and replaces $x$ with a {\tt NaN} if this is not the case; leaves $x$ intact otherwise (i.e., if $x$ is an unsigned byte or a {\tt NaN}).
 \end{itemize}
 
 \mysubsection{Comparison primitives}

lite-client/lite-client.cpp

@@ -4261,6 +4261,8 @@ int main(int argc, char* argv[]) {
   });
   p.add_option('p', "pub", "remote public key",
                [&](td::Slice arg) { td::actor::send_closure(x, &TestNode::set_public_key, td::BufferSlice{arg}); });
+  p.add_option('b', "b64", "remote public key as base64",
+               [&](td::Slice arg) { td::actor::send_closure(x, &TestNode::decode_public_key, td::BufferSlice{arg}); });
   p.add_option('d', "daemonize", "set SIGHUP", [&]() {
     td::set_signal_handler(td::SignalType::HangUp, [](int sig) {
 #if TD_DARWIN || TD_LINUX

lite-client/lite-client.h

@@ -394,6 +394,18 @@ class TestNode : public td::actor::Actor {
     }
     remote_public_key_ = R.move_as_ok();
   }
+  void decode_public_key(td::BufferSlice b64_key) {
+    auto R = [&]() -> td::Result<ton::PublicKey> {
+      std::string key_bytes = {(char)0xc6, (char)0xb4, (char)0x13, (char)0x48};
+      key_bytes = key_bytes + td::base64_decode(b64_key.as_slice().str()).move_as_ok();
+      return ton::PublicKey::import(key_bytes);
+    }();
+
+    if (R.is_error()) {
+      LOG(FATAL) << "bad b64 server public key: " << R.move_as_error();
+    }
+    remote_public_key_ = R.move_as_ok();
+  }
   void set_fail_timeout(td::Timestamp ts) {
     fail_timeout_ = ts;
     alarm_timestamp().relax(fail_timeout_);

overlay/overlay-broadcast.cpp

@@ -17,8 +17,14 @@
     Copyright 2017-2020 Telegram Systems LLP
 */
 #include "overlay-broadcast.hpp"
+#include "adnl/adnl-node-id.hpp"
+#include "common/util.h"
 #include "overlay.hpp"
 #include "keys/encryptor.h"
+#include "td/actor/PromiseFuture.h"
+#include "td/actor/actor.h"
+#include "td/utils/Status.h"
+#include "td/utils/port/Stat.h"
 
 namespace ton {
 
@@ -33,7 +39,13 @@ td::Status BroadcastSimple::check_duplicate() {
 }
 
 td::Status BroadcastSimple::check_source() {
-  return overlay_->check_source_eligible(source_, cert_.get(), data_size());
+  auto r = overlay_->check_source_eligible(source_, cert_.get(), data_size(), false);
+  if (r == BroadcastCheckResult::Forbidden) {
+    return td::Status::Error(ErrorCode::error, "broadcast is forbidden");
+  }
+
+  is_valid_ = r == BroadcastCheckResult::Allowed;
+  return td::Status::OK();
 }
 
 td::BufferSlice BroadcastSimple::to_sign() {
@@ -66,6 +78,14 @@ td::Status BroadcastSimple::distribute() {
   return td::Status::OK();
 }
 
+void BroadcastSimple::broadcast_checked(td::Result<td::Unit> R) {
+  if (R.is_error()) {
+    return;
+  }
+  is_valid_ = true;
+  run_continue().ignore();
+}
+
 tl_object_ptr<ton_api::overlay_broadcast> BroadcastSimple::tl() const {
   return create_tl_object<ton_api::overlay_broadcast>(source_.tl(), cert_ ? cert_->tl() : Certificate::empty_tl(),
                                                       flags_, data_.clone(), date_, signature_.clone());
@@ -75,6 +95,25 @@ td::BufferSlice BroadcastSimple::serialize() {
   return serialize_tl_object(tl(), true);
 }
 
+td::Status BroadcastSimple::run_continue() {
+  TRY_STATUS(distribute());
+  deliver();
+  return td::Status::OK();
+}
+
+td::Status BroadcastSimple::run() {
+  TRY_STATUS(run_checks());
+  if (!is_valid_) {
+    auto P = td::PromiseCreator::lambda(
+        [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result<td::Unit> R) mutable {
+          td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(R));
+        });
+    overlay_->check_broadcast(source_.compute_short_id(), data_.clone(), std::move(P));
+    return td::Status::OK();
+  }
+  return run_continue();
+}
+
 td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr<ton_api::overlay_broadcast> broadcast) {
   auto src = PublicKey{broadcast->src_};
   auto data_hash = sha256_bits256(broadcast->data_.as_slice());
@@ -86,7 +125,7 @@ td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr<ton_api::
 
   auto B = std::make_unique<BroadcastSimple>(broadcast_hash, src, std::move(cert), broadcast->flags_,
                                              std::move(broadcast->data_), broadcast->date_,
-                                             std::move(broadcast->signature_), overlay);
+                                             std::move(broadcast->signature_), false, overlay);
   TRY_STATUS(B->run());
   overlay->register_simple_broadcast(std::move(B));
   return td::Status::OK();
@@ -100,7 +139,7 @@ td::Status BroadcastSimple::create_new(td::actor::ActorId<OverlayImpl> overlay,
   auto date = static_cast<td::uint32>(td::Clocks::system());
 
   auto B = std::make_unique<BroadcastSimple>(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date,
-                                             td::BufferSlice{}, nullptr);
+                                             td::BufferSlice{}, false, nullptr);
 
   auto to_sign = B->to_sign();
   auto P = td::PromiseCreator::lambda(

overlay/overlay-broadcast.hpp

@@ -18,9 +18,16 @@
 */
 #pragma once
 
+#include "adnl/adnl-local-id.h"
+#include "adnl/adnl-node-id.hpp"
 #include "auto/tl/ton_api.h"
+#include "common/refcnt.hpp"
 #include "overlay/overlay.h"
+#include "td/actor/PromiseFuture.h"
 #include "td/utils/List.h"
+#include "td/utils/Status.h"
+#include "td/utils/buffer.h"
+#include "td/utils/common.h"
 
 namespace ton {
 
@@ -38,6 +45,7 @@ class BroadcastSimple : public td::ListNode {
   td::BufferSlice data_;
   td::uint32 date_;
   td::BufferSlice signature_;
+  bool is_valid_{false};
 
   OverlayImpl *overlay_;
 
@@ -52,7 +60,7 @@ class BroadcastSimple : public td::ListNode {
 
  public:
   BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr<Certificate> cert,
-                  td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature,
+                  td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid,
                   OverlayImpl *overlay)
       : broadcast_hash_(broadcast_hash)
       , source_(std::move(source))
@@ -61,6 +69,7 @@ class BroadcastSimple : public td::ListNode {
       , data_(std::move(data))
       , date_(date)
       , signature_(std::move(signature))
+      , is_valid_(is_valid)
       , overlay_(overlay) {
   }
 
@@ -80,17 +89,14 @@ class BroadcastSimple : public td::ListNode {
   }
   void deliver();
 
-  td::Status run() {
+  td::Status run();
-    TRY_STATUS(run_checks());
+  td::Status run_continue();
-    TRY_STATUS(distribute());
-    deliver();
-    return td::Status::OK();
-  }
 
   tl_object_ptr<ton_api::overlay_broadcast> tl() const;
   td::BufferSlice serialize();
 
   void update_overlay(OverlayImpl *overlay);
+  void broadcast_checked(td::Result<td::Unit> R);
 
   static td::Status create(OverlayImpl *overlay, tl_object_ptr<ton_api::overlay_broadcast> broadcast);
   static td::Status create_new(td::actor::ActorId<OverlayImpl> overlay, td::actor::ActorId<keyring::Keyring> keyring,

overlay/overlay-fec-broadcast.cpp

@@ -54,10 +54,20 @@ td::Status OverlayFecBroadcastPart::check_duplicate() {
 }
 
 td::Status OverlayFecBroadcastPart::check_source() {
-  TRY_STATUS(overlay_->check_source_eligible(source_, cert_.get(), broadcast_size_));
+  auto r = overlay_->check_source_eligible(source_, cert_.get(), broadcast_size_, true);
+  if (r == BroadcastCheckResult::Forbidden) {
+    return td::Status::Error(ErrorCode::error, "broadcast is forbidden");
+  }
+
+  if (r == BroadcastCheckResult::NeedCheck) {
+    untrusted_ = true;
+    return td::Status::OK();
+  }
+
   if (bcast_) {
     TRY_STATUS(bcast_->is_eligible_sender(source_));
   }
+
   return td::Status::OK();
 }
 
@@ -68,6 +78,7 @@ td::Status OverlayFecBroadcastPart::check_signature() {
 }
 
 td::Status OverlayFecBroadcastPart::run_checks() {
+
   TRY_STATUS(check_time());
   TRY_STATUS(check_duplicate());
   TRY_STATUS(check_source());
@@ -75,7 +86,52 @@ td::Status OverlayFecBroadcastPart::run_checks() {
   return td::Status::OK();
 }
 
+void BroadcastFec::broadcast_checked(td::Result<td::Unit> R) {
+  if (R.is_error()) {
+    return;
+  }
+  overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone());
+  auto manager = overlay_->overlay_manager();
+  while (!parts_.empty()) {
+       distribute_part(parts_.begin()->first);
+  }
+}
+
+// Do we need status here??
+td::Status  BroadcastFec::distribute_part(td::uint32 seqno) {
+  auto i = parts_.find(seqno);
+  if (i == parts_.end()) {
+    // should not get here
+    return td::Status::OK();
+  }
+  auto tls = std::move(i->second);
+  parts_.erase(i);
+  td::BufferSlice data_short = std::move(tls.first);
+  td::BufferSlice data = std::move(tls.second);
+
+  auto nodes = overlay_->get_neighbours(5);
+  auto manager = overlay_->overlay_manager();
+
+  for (auto &n : nodes) {
+    if (neighbour_completed(n)) {
+      continue;
+    }
+    if (neighbour_received(n)) {
+      td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(),
+                              data_short.clone());
+    } else {
+      if (hash_.count_leading_zeroes() >= 12) {
+        VLOG(OVERLAY_INFO) << "broadcast " << hash_ << ": sending part " << seqno << " to " << n;
+      }
+      td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(),
+                              data.clone());
+    }
+  }
+  return td::Status::OK();
+}
+
 td::Status OverlayFecBroadcastPart::apply() {
+
   if (!bcast_) {
     bcast_ = overlay_->get_fec_broadcast(broadcast_hash_);
   }
@@ -98,52 +154,31 @@ td::Status OverlayFecBroadcastPart::apply() {
   }
 
   if (!bcast_->finalized()) {
-    TRY_STATUS(bcast_->add_part(seqno_, data_.clone()));
+    bcast_->set_overlay(overlay_);
+    TRY_STATUS(bcast_->add_part(seqno_, data_.clone(), export_serialized_short(), export_serialized()));
     auto R = bcast_->finish();
     if (R.is_error()) {
       auto S = R.move_as_error();
       if (S.code() != ErrorCode::notready) {
         return S;
       }
+    } else {
+      if(untrusted_) {
+        auto P = td::PromiseCreator::lambda(
+              [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result<td::Unit> RR) mutable {
+                td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(RR));
+              });
+        overlay_->check_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok(), std::move(P));
       } else {
         overlay_->deliver_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok());
       }
     }
-
+  }
   return td::Status::OK();
 }
 
 td::Status OverlayFecBroadcastPart::distribute() {
-  auto B = export_serialized();
+  TRY_STATUS(bcast_->distribute_part(seqno_));
-  auto nodes = overlay_->get_neighbours(5);
-
-  auto manager = overlay_->overlay_manager();
-
-  td::BufferSlice data;
-  td::BufferSlice data_short;
-
-  for (auto &n : nodes) {
-    if (bcast_->neighbour_completed(n)) {
-      continue;
-    }
-    if (bcast_->neighbour_received(n)) {
-      if (data_short.size() == 0) {
-        data_short = export_serialized_short();
-      }
-      td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(),
-                              data_short.clone());
-    } else {
-      if (data.size() == 0) {
-        data = export_serialized();
-      }
-
-      if (broadcast_hash_.count_leading_zeroes() >= 12) {
-        VLOG(OVERLAY_INFO) << "broadcast " << broadcast_hash_ << ": sending part " << part_hash_ << " to " << n;
-      }
-      td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(),
-                              data.clone());
-    }
-  }
   return td::Status::OK();
 }
 
@@ -179,7 +214,6 @@ td::BufferSlice OverlayFecBroadcastPart::to_sign() {
 td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay,
                                            tl_object_ptr<ton_api::overlay_broadcastFec> broadcast) {
   TRY_STATUS(overlay->check_date(broadcast->date_));
-
   auto source = PublicKey{broadcast->src_};
   auto part_data_hash = sha256_bits256(broadcast->data_.as_slice());
 

overlay/overlay-fec-broadcast.hpp

@@ -82,13 +82,17 @@ class BroadcastFec : public td::ListNode {
     }
   }
 
-  td::Status add_part(td::uint32 seqno, td::BufferSlice data) {
+  td::Status add_part(td::uint32 seqno, td::BufferSlice data,
+                      td::BufferSlice serialized_fec_part_short,
+                      td::BufferSlice serialized_fec_part) {
     CHECK(decoder_);
     td::fec::Symbol s;
     s.id = seqno;
     s.data = std::move(data);
 
     decoder_->add_symbol(std::move(s));
+    parts_[seqno] = std::pair<td::BufferSlice, td::BufferSlice>(std::move(serialized_fec_part_short),
+                                                                std::move(serialized_fec_part));
 
     return td::Status::OK();
   }
@@ -106,6 +110,7 @@ class BroadcastFec : public td::ListNode {
     CHECK(encoder_ != nullptr);
     ready_ = true;
     decoder_ = nullptr;
+    data_ = D.data.clone();
     return std::move(D.data);
   }
 
@@ -185,6 +190,13 @@ class BroadcastFec : public td::ListNode {
     }
   }
 
+  void broadcast_checked(td::Result<td::Unit> R);
+  void set_overlay(OverlayImpl *overlay) {
+    overlay_ = overlay;
+  }
+
+  td::Status distribute_part(td::uint32 seqno);
+
  private:
   bool ready_ = false;
 
@@ -205,6 +217,10 @@ class BroadcastFec : public td::ListNode {
 
   td::uint32 next_seqno_ = 0;
   td::uint64 received_parts_ = 0;
+
+  std::map<td::uint32, std::pair<td::BufferSlice, td::BufferSlice>> parts_;
+  OverlayImpl *overlay_;
+  td::BufferSlice data_;
 };
 
 class OverlayFecBroadcastPart : public td::ListNode {
@@ -225,6 +241,7 @@ class OverlayFecBroadcastPart : public td::ListNode {
   td::BufferSlice signature_;
 
   bool is_short_;
+  bool untrusted_{false};
 
   BroadcastFec *bcast_;
   OverlayImpl *overlay_;
@@ -287,7 +304,9 @@ class OverlayFecBroadcastPart : public td::ListNode {
   td::Status run() {
     TRY_STATUS(run_checks());
     TRY_STATUS(apply());
+    if(!untrusted_) {
       TRY_STATUS(distribute());
+    }
     return td::Status::OK();
   }
 
@@ -311,4 +330,3 @@ class OverlayFecBroadcastPart : public td::ListNode {
 }  // namespace overlay
 
 }  // namespace ton
-

overlay/overlay-manager.cpp

@@ -17,16 +17,22 @@
     Copyright 2017-2020 Telegram Systems LLP
 */
 #include "overlay-manager.h"
+#include "auto/tl/ton_api.h"
 #include "overlay.h"
 
 #include "adnl/utils.hpp"
+#include "td/actor/actor.h"
+#include "td/actor/common.h"
 #include "td/utils/Random.h"
 
 #include "td/db/RocksDb.h"
 
+#include "td/utils/Status.h"
 #include "td/utils/overloaded.h"
 
 #include "keys/encryptor.h"
+#include "td/utils/port/Poll.h"
+#include <vector>
 
 namespace ton {
 
@@ -268,17 +274,67 @@ void OverlayManager::save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort o
   db_.set(key, create_serialize_tl_object<ton_api::overlay_db_nodes>(std::move(obj)));
 }
 
-Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature)
+void OverlayManager::get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlaysStats>> promise) {
+  class Cb : public td::actor::Actor {
+   public:
+    Cb(td::Promise<tl_object_ptr<ton_api::engine_validator_overlaysStats>> promise) : promise_(std::move(promise)) {
+    }
+    void incr_pending() {
+      pending_++;
+    }
+    void decr_pending() {
+      if (!--pending_) {
+        promise_.set_result(create_tl_object<ton_api::engine_validator_overlaysStats>(std::move(res_)));
+        stop();
+      }
+    }
+    void receive_answer(tl_object_ptr<ton_api::engine_validator_overlayStats> res) {
+      if (res) {
+        res_.push_back(std::move(res));
+      }
+      decr_pending();
+    }
+
+   private:
+    std::vector<tl_object_ptr<ton_api::engine_validator_overlayStats>> res_;
+    size_t pending_{1};
+    td::Promise<tl_object_ptr<ton_api::engine_validator_overlaysStats>> promise_;
+  };
+
+  auto act = td::actor::create_actor<Cb>("overlaysstatsmerger", std::move(promise)).release();
+
+  for (auto &a : overlays_) {
+    for (auto &b : a.second) {
+      td::actor::send_closure(act, &Cb::incr_pending);
+      td::actor::send_closure(b.second, &Overlay::get_stats,
+                              [act](td::Result<tl_object_ptr<ton_api::engine_validator_overlayStats>> R) {
+                                if (R.is_ok()) {
+                                  td::actor::send_closure(act, &Cb::receive_answer, R.move_as_ok());
+                                } else {
+                                  td::actor::send_closure(act, &Cb::receive_answer, nullptr);
+                                }
+                              });
+    }
+  }
+
+  td::actor::send_closure(act, &Cb::decr_pending);
+}
+
+Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags,
+                         td::BufferSlice signature)
     : issued_by_(issued_by)
     , expire_at_(expire_at)
     , max_size_(max_size)
+    , flags_(flags)
     , signature_(td::SharedSlice(signature.as_slice())) {
 }
 
-Certificate::Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature)
+Certificate::Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags,
+                         td::BufferSlice signature)
     : issued_by_(issued_by)
     , expire_at_(expire_at)
     , max_size_(max_size)
+    , flags_(flags)
     , signature_(td::SharedSlice(signature.as_slice())) {
 }
 
@@ -290,9 +346,19 @@ void Certificate::set_issuer(PublicKey issuer) {
   issued_by_ = issuer;
 }
 
+constexpr td::uint32 cert_default_flags(td::uint32 max_size) {
+  return (max_size > Overlays::max_simple_broadcast_size() ? CertificateFlags::AllowFec : 0) |
+         CertificateFlags::Trusted;
+}
+
 td::BufferSlice Certificate::to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const {
+  if (flags_ == cert_default_flags(max_size_)) {
     return create_serialize_tl_object<ton_api::overlay_certificateId>(overlay_id.tl(), issued_to.tl(), expire_at_,
                                                                       max_size_);
+  } else {
+    return create_serialize_tl_object<ton_api::overlay_certificateIdV2>(overlay_id.tl(), issued_to.tl(), expire_at_,
+                                                                        max_size_, flags_);
+  }
 }
 
 const PublicKeyHash Certificate::issuer_hash() const {
@@ -307,32 +373,48 @@ const PublicKey &Certificate::issuer() const {
 
 td::Result<std::shared_ptr<Certificate>> Certificate::create(tl_object_ptr<ton_api::overlay_Certificate> cert) {
   std::shared_ptr<Certificate> res;
-  ton_api::downcast_call(*cert.get(), td::overloaded([&](ton_api::overlay_emptyCertificate &obj) { res = nullptr; },
+  ton_api::downcast_call(*cert.get(),
+                         td::overloaded([&](ton_api::overlay_emptyCertificate &obj) { res = nullptr; },
                                         [&](ton_api::overlay_certificate &obj) {
-                                                       res = std::make_shared<Certificate>(
+                                          res = std::make_shared<Certificate>(PublicKey{obj.issued_by_}, obj.expire_at_,
-                                                           PublicKey{obj.issued_by_}, obj.expire_at_,
                                                                               static_cast<td::uint32>(obj.max_size_),
+                                                                              cert_default_flags(obj.max_size_),
+                                                                              std::move(obj.signature_));
+                                        },
+                                        [&](ton_api::overlay_certificateV2 &obj) {
+                                          res = std::make_shared<Certificate>(PublicKey{obj.issued_by_}, obj.expire_at_,
+                                                                              static_cast<td::uint32>(obj.max_size_),
+                                                                              static_cast<td::uint32>(obj.flags_),
                                                                               std::move(obj.signature_));
                                         }));
   return std::move(res);
 }
 
-td::Status Certificate::check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time,
+BroadcastCheckResult Certificate::check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time,
-                              td::uint32 size) const {
+                                        td::uint32 size, bool is_fec) const {
   if (size > max_size_) {
-    return td::Status::Error(ErrorCode::protoviolation, "too big broadcast size");
+    return BroadcastCheckResult::Forbidden;
   }
   if (unix_time > expire_at_) {
-    return td::Status::Error(ErrorCode::protoviolation, "too old certificate");
+    return BroadcastCheckResult::Forbidden;
+  }
+  if (is_fec && !(flags_ & CertificateFlags::AllowFec)) {
+    return BroadcastCheckResult::Forbidden;
   }
 
-  TRY_RESULT(E, issued_by_.get<PublicKey>().create_encryptor());
+  auto R1 = issued_by_.get<PublicKey>().create_encryptor();
+  if (R1.is_error()) {
+    return BroadcastCheckResult::Forbidden;
+  }
+  auto E = R1.move_as_ok();
 
   auto B = to_sign(overlay_id, node);
 
-  TRY_STATUS(E->check_signature(B.as_slice(), signature_.as_slice()));
+  if (E->check_signature(B.as_slice(), signature_.as_slice()).is_error()) {
+    return BroadcastCheckResult::Forbidden;
+  }
 
-  return td::Status::OK();
+  return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck;
 }
 
 tl_object_ptr<ton_api::overlay_Certificate> Certificate::tl() const {

overlay/overlay-manager.h

@@ -91,6 +91,7 @@ class OverlayManager : public Overlays {
 
   void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id,
                         td::actor::ActorOwn<Overlay> overlay);
+  void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlaysStats>> promise) override;
 
   struct PrintId {};
 

overlay/overlay.cpp

@@ -16,6 +16,7 @@
 
     Copyright 2017-2020 Telegram Systems LLP
 */
+#include "auto/tl/ton_api.h"
 #include "td/utils/Random.h"
 
 #include "adnl/utils.hpp"
@@ -25,6 +26,7 @@
 #include "auto/tl/ton_api.hpp"
 
 #include "keys/encryptor.h"
+#include "td/utils/StringBuilder.h"
 
 namespace ton {
 
@@ -391,25 +393,21 @@ td::Status OverlayImpl::check_date(td::uint32 date) {
   return td::Status::OK();
 }
 
-td::Status OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size) {
+BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size,
+                                                        bool is_fec) {
   if (size == 0) {
-    return td::Status::Error(ErrorCode::protoviolation, "empty broadcast");
+    return BroadcastCheckResult::Forbidden;
   }
   auto short_id = source.compute_short_id();
 
-  auto r = rules_.max_size(source.compute_short_id());
+  auto r = rules_.check_rules(source.compute_short_id(), size, is_fec);
-  if (r >= size) {
+  if (!cert || r == BroadcastCheckResult::Allowed) {
-    return td::Status::OK();
+    return r;
   }
-  if (!cert) {
+
-    return td::Status::Error(ErrorCode::protoviolation, "source is not eligible");
+  auto r2 = cert->check(short_id, overlay_id_, static_cast<td::int32>(td::Clocks::system()), size, is_fec);
-  }
+  r2 = broadcast_check_result_min(r2, rules_.check_rules(cert->issuer_hash(), size, is_fec));
-  TRY_STATUS(cert->check(short_id, overlay_id_, static_cast<td::int32>(td::Clocks::system()), size));
+  return broadcast_check_result_max(r, r2);
-  auto issuer_short = cert->issuer_hash();
-  if (rules_.max_size(issuer_short) < size) {
-    return td::Status::Error(ErrorCode::protoviolation, "bad certificate");
-  }
-  return td::Status::OK();
 }
 
 td::Status OverlayImpl::check_delivered(BroadcastHash hash) {
@@ -539,6 +537,38 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) {
   rules_ = std::move(rules);
 }
 
+void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise<td::Unit> promise) {
+  callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise));
+}
+
+void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result<td::Unit> R) {
+  {
+    auto it = broadcasts_.find(hash);
+    if (it != broadcasts_.end()) {
+      it->second->broadcast_checked(std::move(R));
+    }
+  }
+  {
+    auto it = fec_broadcasts_.find(hash);
+    if (it != fec_broadcasts_.end()) {
+      it->second->broadcast_checked(std::move(R));
+    }
+  }
+}
+
+void OverlayImpl::get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlayStats>> promise) {
+  auto res = create_tl_object<ton_api::engine_validator_overlayStats>();
+  res->adnl_id_ = local_id_.bits256_value();
+  res->overlay_id_ = overlay_id_.bits256_value();
+  res->overlay_id_full_ = id_full_.pubkey().tl();
+  peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { res->nodes_.push_back(key.tl()); });
+
+  res->stats_.push_back(
+      create_tl_object<ton_api::engine_validator_oneStat>("neighbours_cnt", PSTRING() << neighbours_.size()));
+
+  promise.set_value(std::move(res));
+}
+
 }  // namespace overlay
 
 }  // namespace ton

overlay/overlay.h

@@ -63,6 +63,7 @@ class Overlay : public td::actor::Actor {
   virtual void add_certificate(PublicKeyHash key, std::shared_ptr<Certificate>) = 0;
   virtual void set_privacy_rules(OverlayPrivacyRules rules) = 0;
   virtual void receive_nodes_from_db(tl_object_ptr<ton_api::overlay_nodes> nodes) = 0;
+  virtual void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlayStats>> promise) = 0;
   //virtual void receive_broadcast(td::BufferSlice data) = 0;
   //virtual void subscribe(std::unique_ptr<Overlays::Callback> callback) = 0;
 };
@@ -70,4 +71,3 @@ class Overlay : public td::actor::Actor {
 }  // namespace overlay
 
 }  // namespace ton
-

overlay/overlay.hpp

@@ -144,9 +144,12 @@ class OverlayImpl : public Overlay {
   void print(td::StringBuilder &sb) override;
 
   td::Status check_date(td::uint32 date);
-  td::Status check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size);
+  BroadcastCheckResult check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec);
   td::Status check_delivered(BroadcastHash hash);
 
+  void broadcast_checked(Overlay::BroadcastHash hash, td::Result<td::Unit> R);
+  void check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise<td::Unit> promise);
+
   BroadcastFec *get_fec_broadcast(BroadcastHash hash);
   void register_fec_broadcast(std::unique_ptr<BroadcastFec> bcast);
   void register_simple_broadcast(std::unique_ptr<BroadcastSimple> bcast);
@@ -187,6 +190,8 @@ class OverlayImpl : public Overlay {
   std::shared_ptr<Certificate> get_certificate(PublicKeyHash local_id);
   td::Result<Encryptor *> get_encryptor(PublicKey source);
 
+  void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlayStats>> promise) override;
+
  private:
   template <class T>
   void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise<td::BufferSlice> promise) {

overlay/overlays.h

@@ -21,7 +21,11 @@
 #include "adnl/adnl.h"
 #include "dht/dht.h"
 
+#include "td/actor/PromiseFuture.h"
 #include "td/actor/actor.h"
+#include "td/utils/Status.h"
+#include "td/utils/buffer.h"
+#include "td/utils/common.h"
 
 #include <map>
 
@@ -80,41 +84,65 @@ class OverlayIdFull {
   td::BufferSlice name_;
 };
 
+struct CertificateFlags {
+  enum Values : td::uint32 { AllowFec = 1, Trusted = 2 };
+};
+
+enum BroadcastCheckResult { Forbidden = 1, NeedCheck = 2, Allowed = 3 };
+
+inline BroadcastCheckResult broadcast_check_result_max(BroadcastCheckResult l, BroadcastCheckResult r) {
+  return static_cast<BroadcastCheckResult>(std::max(static_cast<td::int32>(l), static_cast<td::int32>(r)));
+}
+inline BroadcastCheckResult broadcast_check_result_min(BroadcastCheckResult l, BroadcastCheckResult r) {
+  return static_cast<BroadcastCheckResult>(std::min(static_cast<td::int32>(l), static_cast<td::int32>(r)));
+}
+
 class OverlayPrivacyRules {
  public:
   OverlayPrivacyRules() {
   }
   OverlayPrivacyRules(td::uint32 size) : max_unath_size_(size) {
   }
-  OverlayPrivacyRules(td::uint32 max_size, std::map<PublicKeyHash, td::uint32> authorized_keys)
+  OverlayPrivacyRules(td::uint32 max_size, td::uint32 flags, std::map<PublicKeyHash, td::uint32> authorized_keys)
-      : max_unath_size_(max_size), authorized_keys_(std::move(authorized_keys)) {
+      : max_unath_size_(max_size), flags_(flags), authorized_keys_(std::move(authorized_keys)) {
   }
 
-  td::uint32 max_size(PublicKeyHash hash) {
+  BroadcastCheckResult check_rules(PublicKeyHash hash, td::uint32 size, bool is_fec) {
+
     auto it = authorized_keys_.find(hash);
     if (it == authorized_keys_.end()) {
-      return max_unath_size_;
+      if (size > max_unath_size_) {
+        return BroadcastCheckResult::Forbidden;
+      }
+      if (!(flags_ & CertificateFlags::AllowFec) && is_fec) {
+        return BroadcastCheckResult::Forbidden;
+      }
+      return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck;
     } else {
-      return it->second;
+      return it->second >= size ? BroadcastCheckResult::Allowed : BroadcastCheckResult::Forbidden;
     }
   }
 
  private:
   td::uint32 max_unath_size_{0};
+  td::uint32 flags_{0};
   std::map<PublicKeyHash, td::uint32> authorized_keys_;
 };
 
 class Certificate {
  public:
-  Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature);
+  Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags,
-  Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature);
+              td::BufferSlice signature);
+  Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags,
+              td::BufferSlice signature);
   Certificate() {
   }
   void set_signature(td::BufferSlice signature);
   void set_issuer(PublicKey issuer);
   td::BufferSlice to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const;
 
-  td::Status check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size) const;
+  BroadcastCheckResult check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size,
+                             bool is_fec) const;
   tl_object_ptr<ton_api::overlay_Certificate> tl() const;
   const PublicKey &issuer() const;
   const PublicKeyHash issuer_hash() const;
@@ -126,6 +154,7 @@ class Certificate {
   td::Variant<PublicKey, PublicKeyHash> issued_by_;
   td::int32 expire_at_;
   td::uint32 max_size_;
+  td::uint32 flags_;
   td::SharedSlice signature_;
 };
 
@@ -137,6 +166,10 @@ class Overlays : public td::actor::Actor {
     virtual void receive_query(adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, td::BufferSlice data,
                                td::Promise<td::BufferSlice> promise) = 0;
     virtual void receive_broadcast(PublicKeyHash src, OverlayIdShort overlay_id, td::BufferSlice data) = 0;
+    virtual void check_broadcast(PublicKeyHash src, OverlayIdShort overlay_id, td::BufferSlice data,
+                                 td::Promise<td::Unit> promise) {
+      promise.set_value(td::Unit());
+    }
     virtual ~Callback() = default;
   };
 
@@ -199,6 +232,7 @@ class Overlays : public td::actor::Actor {
 
   virtual void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers,
                                         td::Promise<std::vector<adnl::AdnlNodeIdShort>> promise) = 0;
+  virtual void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlaysStats>> promise) = 0;
 };
 
 }  // namespace overlay

rldp2/CMakeLists.txt

@@ -51,7 +51,7 @@ target_include_directories(rldp PUBLIC
   ${OPENSSL_INCLUDE_DIR}
 )
 if (GSL_FOUND)
-  target_link_libraries(rldp2 PRIVATE GSL::gsl)
+  target_link_libraries(rldp2 PRIVATE gsl)
   target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1)
 endif()
 target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api)

tdutils/td/utils/DecTree.h

@@ -46,6 +46,17 @@ class DecTree {
       }
     }
 
+    template <typename FuncT>
+    void iterate(const FuncT &cb) {
+      if (left_) {
+        left_->iterate(cb);
+      }
+      cb(key_, value_);
+      if (right_) {
+        right_->iterate(cb);
+      }
+    }
+
     Node(KeyType key, ValueType value, uint32 y) : size_(1), key_(std::move(key)), value_(std::move(value)), y_(y) {
     }
   };
@@ -223,6 +234,15 @@ class DecTree {
   bool exists(const KeyType &key) const {
     return get_node(root_, key) != nullptr;
   }
+
+  template <typename FuncT>
+  void iterate(const FuncT &cb) {
+    if (size() == 0) {
+      return;
+    } else {
+      root_->iterate(cb);
+    }
+  }
 };
 
 }  // namespace td

third-party/rocksdb

@@ -1 +1 @@
-Subproject commit 0915c99f01b46f50af8e02da8b6528156f584b7c
+Subproject commit fcf3d75f3f022a6a55ff1222d6b06f8518d38c7c

tl/generate/scheme/lite_api.tl

@@ -25,6 +25,7 @@ adnl.message.answer query_id:int256 answer:bytes = adnl.Message;
 liteServer.error code:int message:string = liteServer.Error; 
 
 liteServer.accountId workchain:int id:int256 = liteServer.AccountId;
+liteServer.libraryEntry hash:int256 data:bytes = liteServer.LibraryEntry;
 
 liteServer.masterchainInfo last:tonNode.blockIdExt state_root_hash:int256 init:tonNode.zeroStateIdExt = liteServer.MasterchainInfo;
 liteServer.masterchainInfoExt mode:# version:int capabilities:long last:tonNode.blockIdExt last_utime:int now:int state_root_hash:int256 init:tonNode.zeroStateIdExt = liteServer.MasterchainInfoExt;
@@ -50,6 +51,7 @@ liteServer.blockLinkForward to_key_block:Bool from:tonNode.blockIdExt to:tonNode
 liteServer.partialBlockProof complete:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt steps:(vector liteServer.BlockLink) = liteServer.PartialBlockProof;
 liteServer.configInfo mode:# id:tonNode.blockIdExt state_proof:bytes config_proof:bytes = liteServer.ConfigInfo;
 liteServer.validatorStats mode:# id:tonNode.blockIdExt count:int complete:Bool state_proof:bytes data_proof:bytes = liteServer.ValidatorStats;
+liteServer.libraryResult result:(vector liteServer.libraryEntry) = liteServer.LibraryResult;
 
 liteServer.debug.verbosity value:int = liteServer.debug.Verbosity;
 
@@ -75,6 +77,7 @@ liteServer.getBlockProof mode:# known_block:tonNode.blockIdExt target_block:mode
 liteServer.getConfigAll mode:# id:tonNode.blockIdExt = liteServer.ConfigInfo;
 liteServer.getConfigParams mode:# id:tonNode.blockIdExt param_list:(vector int) = liteServer.ConfigInfo;
 liteServer.getValidatorStats#091a58bc mode:# id:tonNode.blockIdExt limit:int start_after:mode.0?int256 modified_after:mode.2?int = liteServer.ValidatorStats;
+liteServer.getLibraries library_list:(vector int256) = liteServer.LibraryResult;
 
 liteServer.queryPrefix = Object; 
 liteServer.query data:bytes = Object;

tl/generate/scheme/ton_api.tl

@@ -219,9 +219,11 @@ overlay.broadcastFec.partId broadcast_hash:int256 data_hash:int256 seqno:int = o
 overlay.broadcast.toSign hash:int256 date:int = overlay.broadcast.ToSign;
 
 overlay.certificate issued_by:PublicKey expire_at:int max_size:int signature:bytes = overlay.Certificate;
+overlay.certificateV2 issued_by:PublicKey expire_at:int max_size:int flags:int signature:bytes = overlay.Certificate;
 overlay.emptyCertificate = overlay.Certificate;
 
 overlay.certificateId overlay_id:int256 node:int256 expire_at:int max_size:int = overlay.CertificateId;
+overlay.certificateIdV2 overlay_id:int256 node:int256 expire_at:int max_size:int flags:int = overlay.CertificateId;
 
 overlay.unicast data:bytes = overlay.Broadcast;
 overlay.broadcast src:PublicKey certificate:overlay.Certificate flags:int data:bytes date:int signature:bytes = overlay.Broadcast;
@@ -618,6 +620,11 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P
 engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus;
 engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus;
 
+engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 nodes:(vector adnl.id.short) 
+                              stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats;
+engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats;
+
+
 ---functions---
 
 engine.validator.getTime = engine.validator.Time;
@@ -659,8 +666,14 @@ engine.validator.createComplaintVote election_id:int vote:bytes = engine.validat
 
 engine.validator.checkDhtServers id:int256 = engine.validator.DhtServersStatus;
 
+engine.validator.getOverlaysStats = engine.validator.OverlaysStats;
+
 engine.validator.controlQuery data:bytes = Object;
 
+engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success;
+engine.validator.signShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash expire_at:int max_size:int = overlay.Certificate;
+engine.validator.importShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success;
+
 ---types---
 
 storage.pong = storage.Pong;

tl/generate/scheme/tonlib_api.tl

@@ -208,8 +208,11 @@ blocks.shards shards:vector<ton.BlockIdExt> = blocks.Shards;
 blocks.accountTransactionId account:bytes lt:int64 = blocks.AccountTransactionId;
 blocks.shortTxId mode:# account:mode.0?bytes lt:mode.1?int64 hash:mode.2?bytes = liteServer.TransactionId;
 blocks.transactions id:ton.blockIdExt req_count:int32 incomplete:Bool transactions:vector<blocks.shortTxId> = blocks.Transactions;
-blocks.header id:ton.blockIdExt global_id:int32 version:int32 after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 vert_seqno:# prev_blocks:vector<ton.blockIdExt> = blocks.Header;
+blocks.header id:ton.blockIdExt global_id:int32 version:int32 flags:# after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 gen_utime:int53 vert_seqno:# prev_blocks:vector<ton.blockIdExt> = blocks.Header;
 //blocks.shortData header:blocks.Header transactions:blocks.Header = blocks.BlockData;
+
+configInfo config:tvm.cell = ConfigInfo;
+
 ---functions---
 
 init options:options = options.Info;
@@ -262,6 +265,8 @@ guessAccount public_key:string rwallet_init_public_key:string = AccountRevisionL
 getAccountState account_address:accountAddress = FullAccountState;
 createQuery private_key:InputKey address:accountAddress timeout:int32 action:Action initial_account_state:InitialAccountState = query.Info;
 
+getConfigParam mode:# id:ton.blockIdExt param:# = ConfigInfo;
+
 msg.decrypt input_key:InputKey data:msg.dataEncryptedArray = msg.DataDecryptedArray;
 msg.decryptWithProof proof:bytes data:msg.dataEncrypted = msg.Data;
 

tonlib/tonlib/LastConfig.cpp

@@ -62,9 +62,7 @@ void LastConfig::with_last_block(td::Result<LastBlockState> r_last_block) {
   }
 
   auto last_block = r_last_block.move_as_ok();
-  auto params = params_;
+  client_.send_query(ton::lite_api::liteServer_getConfigAll(0, create_tl_lite_block_id(last_block.last_block_id)),
-  client_.send_query(ton::lite_api::liteServer_getConfigParams(0, create_tl_lite_block_id(last_block.last_block_id),
-                                                               std::move(params)),
                      [this](auto r_config) { this->on_config(std::move(r_config)); });
 }
 

tonlib/tonlib/TonlibClient.cpp

@@ -861,13 +861,13 @@ class Query {
     }
     return res;
   }
-  td::Result<std::pair<Fee, std::vector<Fee>>> estimate_fees(bool ignore_chksig, const block::Config& cfg) {
+  td::Result<std::pair<Fee, std::vector<Fee>>> estimate_fees(bool ignore_chksig, std::shared_ptr<const block::Config>& cfg, vm::Dictionary& libraries) {
     // gas fees
     bool is_masterchain = raw_.source->get_address().workchain == ton::masterchainId;
-    TRY_RESULT(gas_limits_prices, cfg.get_gas_limits_prices(is_masterchain));
+    TRY_RESULT(gas_limits_prices, cfg->get_gas_limits_prices(is_masterchain));
-    TRY_RESULT(storage_prices, cfg.get_storage_prices());
+    TRY_RESULT(storage_prices, cfg->get_storage_prices());
-    TRY_RESULT(masterchain_msg_prices, cfg.get_msg_prices(true));
+    TRY_RESULT(masterchain_msg_prices, cfg->get_msg_prices(true));
-    TRY_RESULT(basechain_msg_prices, cfg.get_msg_prices(false));
+    TRY_RESULT(basechain_msg_prices, cfg->get_msg_prices(false));
     block::MsgPrices* msg_prices[2] = {&basechain_msg_prices, &masterchain_msg_prices};
     auto storage_fee_256 = block::StoragePrices::compute_storage_fees(
         raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_stat,
@@ -888,7 +888,9 @@ class Query {
                                                                         .set_limits(gas_limits)
                                                                         .set_balance(raw_.source->get_balance())
                                                                         .set_now(raw_.source->get_sync_time())
-                                                                        .set_ignore_chksig(ignore_chksig));
+                                                                        .set_ignore_chksig(ignore_chksig)
+                                                                        .set_address(raw_.source->get_address())
+                                                                        .set_config(cfg).set_libraries(libraries));
     td::int64 fwd_fee = 0;
     if (res.success) {
       LOG(DEBUG) << "output actions:\n"
@@ -910,7 +912,7 @@ class Query {
 
     for (auto& destination : raw_.destinations) {
       bool dest_is_masterchain = destination && destination->get_address().workchain == ton::masterchainId;
-      TRY_RESULT(dest_gas_limits_prices, cfg.get_gas_limits_prices(dest_is_masterchain));
+      TRY_RESULT(dest_gas_limits_prices, cfg->get_gas_limits_prices(dest_is_masterchain));
       auto dest_storage_fee_256 =
           destination ? block::StoragePrices::compute_storage_fees(
                             destination->get_sync_time(), storage_prices, destination->raw().storage_stat,
@@ -1887,6 +1889,8 @@ td::Status TonlibClient::do_request(const tonlib_api::init& request,
   TRY_RESULT(kv, std::move(r_kv));
   kv_ = std::shared_ptr<KeyValue>(kv.release());
 
+  load_libs_from_disk();
+
   key_storage_.set_key_value(kv_);
   last_block_storage_.set_key_value(kv_);
   auto res = tonlib_api::make_object<tonlib_api::options_info>();
@@ -2306,9 +2310,10 @@ struct ToRawTransactions {
           return td::Status::Error("Failed to unpack CommonMsgInfo::ext_out_msg_info");
         }
         TRY_RESULT(src, to_std_address(msg_info.src));
+        auto created_lt = static_cast<td::int64>(msg_info.created_lt);
         return tonlib_api::make_object<tonlib_api::raw_message>(
             tonlib_api::make_object<tonlib_api::accountAddress>(src),
-            tonlib_api::make_object<tonlib_api::accountAddress>(), 0, 0, 0, 0, std::move(body_hash), get_data(src));
+            tonlib_api::make_object<tonlib_api::accountAddress>(), 0, 0, 0, created_lt, std::move(body_hash), get_data(src));
       }
     }
 
@@ -2353,7 +2358,7 @@ struct ToRawTransactions {
 
       if (trans.outmsg_cnt != 0) {
         vm::Dictionary dict{trans.r1.out_msgs, 15};
-        for (int x = 0; x < trans.outmsg_cnt && x < 100; x++) {
+        for (int x = 0; x < trans.outmsg_cnt; x++) {
           TRY_RESULT(out_msg, to_raw_message(dict.lookup_ref(td::BitArray<15>{x})));
           fees += out_msg->fwd_fee_;
           fees += out_msg->ihr_fee_;
@@ -3266,7 +3271,7 @@ void TonlibClient::query_estimate_fees(td::int64 id, bool ignore_chksig, td::Res
     return;
   }
   TRY_RESULT_PROMISE(promise, state, std::move(r_state));
-  TRY_RESULT_PROMISE_PREFIX(promise, fees, TRY_VM(it->second->estimate_fees(ignore_chksig, *state.config)),
+  TRY_RESULT_PROMISE_PREFIX(promise, fees, TRY_VM(it->second->estimate_fees(ignore_chksig, state.config, libraries)),
                             TonlibError::Internal());
   promise.set_value(tonlib_api::make_object<tonlib_api::query_fees>(
       fees.first.to_tonlib_api(), td::transform(fees.second, [](auto& x) { return x.to_tonlib_api(); })));
@@ -3440,7 +3445,8 @@ td::Result<vm::StackEntry> from_tonlib_api(tonlib_api::tvm_StackEntry& entry) {
           [&](tonlib_api::tvm_stackEntryUnsupported& cell) { return td::Status::Error("Unsuppored stack entry"); },
           [&](tonlib_api::tvm_stackEntrySlice& cell) -> td::Result<vm::StackEntry> {
             TRY_RESULT(res, vm::std_boc_deserialize(cell.slice_->bytes_));
-            return vm::StackEntry{std::move(res)};
+            auto slice = vm::load_cell_slice_ref(std::move(res));
+            return vm::StackEntry{std::move(slice)};
           },
           [&](tonlib_api::tvm_stackEntryCell& cell) -> td::Result<vm::StackEntry> {
             TRY_RESULT(res, vm::std_boc_deserialize(cell.cell_->bytes_));
@@ -3472,6 +3478,38 @@ td::Result<vm::StackEntry> from_tonlib_api(tonlib_api::tvm_StackEntry& entry) {
           }));
 }
 
+void deep_library_search(std::set<td::Bits256>& set, std::set<vm::Cell::Hash>& visited,
+                         vm::Dictionary libs, td::Ref<vm::Cell> cell, int depth) {
+  if (depth <= 0 || set.size() >= 16 || visited.size() >= 256) {
+    return;
+  }
+  auto ins = visited.insert(cell->get_hash());
+  if (!ins.second) {
+    return;  // already visited this cell
+  }
+  auto r_loaded_cell = cell->load_cell();
+  if (r_loaded_cell.is_error()) {
+    return;
+  }
+  auto loaded_cell = r_loaded_cell.move_as_ok();
+  if (loaded_cell.data_cell->is_special()) {
+    if (loaded_cell.data_cell->special_type() == vm::DataCell::SpecialType::Library) {
+      vm::CellSlice cs(std::move(loaded_cell));
+      if (cs.size() != vm::Cell::hash_bits + 8) {
+        return;
+      }
+      auto key = td::Bits256(cs.data_bits() + 8);
+      if (libs.lookup(key).is_null()) {
+        set.insert(key);
+      }
+    }
+    return;
+  }
+  for (unsigned int i=0; i<loaded_cell.data_cell->get_refs_cnt(); i++) {
+    deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1);
+  }
+}
+
 td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request,
                                     td::Promise<object_ptr<tonlib_api::smc_runResult>>&& promise) {
   auto it = smcs_.find(request.id_);
@@ -3493,15 +3531,120 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request,
   args.set_stack(std::move(stack));
   args.set_balance(it->second->get_balance());
   args.set_now(it->second->get_sync_time());
-  auto res = smc->run_get_method(std::move(args));
+  args.set_address(it->second->get_address());
+
+  client_.with_last_config([self = this, smc = std::move(smc), args = std::move(args), promise = std::move(promise)
+  ](td::Result<LastConfigState> r_state) mutable {
+    TRY_RESULT_PROMISE(promise, state, std::move(r_state));
+    args.set_config(state.config);
+
+    auto code = smc->get_state().code;
+    if (code.not_null()) {
+      std::set<td::Bits256> librarySet;
+      std::set<vm::Cell::Hash> visited;
+      deep_library_search(librarySet, visited, self->libraries, code, 24);
+      std::vector<td::Bits256> libraryList{librarySet.begin(), librarySet.end()};
+      if (libraryList.size() > 0) {
+        LOG(DEBUG) << "Requesting found libraries in code (" << libraryList.size() << ")";
+        self->client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(libraryList)),
+                    [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)]
+                    (td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) mutable
+        {
+          if (r_libraries.is_error()) {
+            LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string();
+          } else {
+            auto libraries = r_libraries.move_as_ok();
+            bool updated = false;
+            for (auto& lr : libraries->result_) {
+              auto contents = vm::std_boc_deserialize(lr->data_);
+              if (contents.is_ok() && contents.ok().not_null()) {
+                if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) {
+                  LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex();
+                  continue;
+                }
+                self->libraries.set_ref(lr->hash_, contents.move_as_ok());
+                updated = true;
+                LOG(DEBUG) << "registered library " << lr->hash_.to_hex();
+              } else {
+                LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex();
+              }
+            }
+            if (updated) {
+              self->store_libs_to_disk();
+            }
+          }
+          self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise));
+        });
+      }
+      else {
+        self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise));
+      }
+    }
+    else {
+      self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise));
+    }
+  });
+  return td::Status::OK();
+}
+
+void TonlibClient::perform_smc_execution(td::Ref<ton::SmartContract> smc, ton::SmartContract::Args args,
+                                         td::Promise<object_ptr<tonlib_api::smc_runResult>>&& promise) {
+
+  args.set_libraries(libraries);
+
+  auto res = smc->run_get_method(args);
 
   // smc.runResult gas_used:int53 stack:vector<tvm.StackEntry> exit_code:int32 = smc.RunResult;
   std::vector<object_ptr<tonlib_api::tvm_StackEntry>> res_stack;
   for (auto& entry : res.stack->as_span()) {
     res_stack.push_back(to_tonlib_api(entry));
   }
+
+  if (res.missing_library.not_null()) {
+    td::Bits256 hash = res.missing_library;
+    LOG(DEBUG) << "Requesting missing library: " << hash.to_hex();
+    std::vector<td::Bits256> req = {std::move(hash)};
+    client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(req)),
+                [self = this, res = std::move(res), res_stack = std::move(res_stack), hash = std::move(hash),
+                 smc = std::move(smc), args = std::move(args), promise = std::move(promise)]
+                (td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) mutable
+    {
+      if (r_libraries.is_error()) {
+        LOG(WARNING) << "cannot obtain missing library: " << r_libraries.move_as_error().to_string();
         promise.set_value(tonlib_api::make_object<tonlib_api::smc_runResult>(res.gas_used, std::move(res_stack), res.code));
-  return td::Status::OK();
+        return;
+      }
+      bool found = false, updated = false;
+      auto libraries = r_libraries.move_as_ok();
+      for (auto& lr : libraries->result_) {
+        auto contents = vm::std_boc_deserialize(lr->data_);
+        if (contents.is_ok() && contents.ok().not_null()) {
+          if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) {
+            LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex();
+            continue;
+          }
+          found |= (lr->hash_ == hash);
+          updated = true;
+          self->libraries.set_ref(lr->hash_, contents.move_as_ok());
+          LOG(DEBUG) << "registered library " << lr->hash_.to_hex();
+        } else {
+          LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex();
+        }
+      }
+      if (updated) {
+        self->store_libs_to_disk();
+      }
+      if (!found) {
+        LOG(WARNING) << "cannot obtain library " << hash.to_hex() << ", it may not exist";
+        promise.set_value(tonlib_api::make_object<tonlib_api::smc_runResult>(res.gas_used, std::move(res_stack), res.code));
+      } else {
+        self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise));
+      }
+    });
+  }
+  else {
+    promise.set_value(tonlib_api::make_object<tonlib_api::smc_runResult>(res.gas_used, std::move(res_stack), res.code));
+  }
 }
 
 td::Result<tonlib_api::object_ptr<tonlib_api::dns_EntryData>> to_tonlib_api(
@@ -4057,6 +4200,32 @@ auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result<lite_api_p
       blk.workchain_, blk.shard_, blk.seqno_, root_hash, file_hash);
 }
 
+td::Status TonlibClient::do_request(const tonlib_api::getConfigParam& request,
+                        td::Promise<object_ptr<tonlib_api::configInfo>>&& promise) {
+  TRY_RESULT(lite_block, to_lite_api(*request.id_))
+  auto block = create_block_id(std::move(lite_block));
+  auto param = request.param_;
+  std::vector<int32_t> params = { param };
+
+  client_.send_query(ton::lite_api::liteServer_getConfigParams(0, std::move(lite_block), std::move(params)),
+                     promise.wrap([block, param](auto r_config) { 
+    auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(),
+                                                  r_config->config_proof_.as_slice());
+    if (state.is_error()) {
+      LOG(ERROR) << "block::check_extract_state_proof failed: " << state.error();
+    }
+    auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0);
+    if (config.is_error()) {
+      LOG(ERROR) << "block::Config::extract_from_state failed: " << config.error();
+    }
+    tonlib_api::configInfo config_result;
+    config_result.config_ = tonlib_api::make_object<tonlib_api::tvm_cell>(to_bytes(config.move_as_ok()->get_config_param(param)));
+    return tonlib_api::make_object<tonlib_api::configInfo>(std::move(config_result));
+  }));
+
+  return td::Status::OK();
+}
+
 td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainInfo& masterchain_info,
                         td::Promise<object_ptr<tonlib_api::blocks_masterchainInfo>>&& promise) {
   client_.send_query(ton::lite_api::liteServer_getMasterchainInfo(),
@@ -4190,6 +4359,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req
                                   header.id_ = to_tonlib_api(blk_id);
                                   header.global_id_ = blk.global_id;
                                   header.version_ = info.version;
+                                  header.flags_ = info.flags;
                                   header.after_merge_ = info.after_merge;
                                   header.after_split_ = info.after_split;
                                   header.before_split_ = info.before_split;
@@ -4200,6 +4370,8 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req
                                   header.min_ref_mc_seqno_ = info.min_ref_mc_seqno;
                                   header.start_lt_ = info.start_lt;
                                   header.end_lt_ = info.end_lt;
+                                  header.gen_utime_ = info.gen_utime;
+                                  header.is_key_block_ = info.key_block;
                                   header.vert_seqno_ = info.vert_seq_no;
                                   if(!info.not_master) {
                                    header.prev_key_block_seqno_ = info.prev_key_block_seqno;
@@ -4229,6 +4401,28 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req
   return td::Status::OK();
 }
 
+void TonlibClient::load_libs_from_disk() {
+  LOG(DEBUG) << "loading libraries from disk cache";
+  auto r_data = kv_->get("tonlib.libcache");
+  if (r_data.is_error()) {
+    return;
+  }
+  auto r_dict = vm::std_boc_deserialize(r_data.move_as_ok(), true);
+  if (r_dict.is_error()) {
+    return;
+  }
+  libraries = vm::Dictionary(vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice(
+                                                                   r_dict.move_as_ok())).finalize()), 256);
+  // int n = 0; for (auto&& lr : libraries) n++;
+  LOG(DEBUG) << "loaded libraries from disk cache";
+}
+
+void TonlibClient::store_libs_to_disk() {  // NB: Dictionary.get_root_cell does not compute_root, and it is protected
+  kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root())
+                                                        .finalize()).move_as_ok().as_slice());
+  // int n = 0; for (auto&& lr : libraries) n++;
+  LOG(DEBUG) << "stored libraries to disk cache";
+}
 
 template <class P>
 td::Status TonlibClient::do_request(const tonlib_api::runTests& request, P&&) {

tonlib/tonlib/TonlibClient.h

@@ -107,6 +107,7 @@ class TonlibClient : public td::actor::Actor {
     td::optional<ton::BlockIdExt> block_id;
   };
   QueryContext query_context_;
+  vm::Dictionary libraries{256};
 
   // network
   td::actor::ActorOwn<ton::adnl::AdnlExtClient> raw_client_;
@@ -321,7 +322,8 @@ class TonlibClient : public td::actor::Actor {
   td::Status do_request(tonlib_api::pchan_unpackPromise& request,
                         td::Promise<object_ptr<tonlib_api::pchan_promise>>&& promise);
 
-
+  void perform_smc_execution(td::Ref<ton::SmartContract> smc, ton::SmartContract::Args args,
+                             td::Promise<object_ptr<tonlib_api::smc_runResult>>&& promise);
 
   void do_dns_request(std::string name, td::int32 category, td::int32 ttl, td::optional<ton::BlockIdExt> block_id,
                       block::StdAddress address, td::Promise<object_ptr<tonlib_api::dns_resolved>>&& promise);
@@ -355,8 +357,14 @@ class TonlibClient : public td::actor::Actor {
   td::Status do_request(const tonlib_api::blocks_getBlockHeader& request,
                         td::Promise<object_ptr<tonlib_api::blocks_header>>&& promise);
 
+  td::Status do_request(const tonlib_api::getConfigParam& request,
+                        td::Promise<object_ptr<tonlib_api::configInfo>>&& promise);
+
   void proxy_request(td::int64 query_id, std::string data);
 
+  void load_libs_from_disk();
+  void store_libs_to_disk();
+
   friend class TonlibQueryActor;
   struct Target {
     bool can_be_empty{true};

validator-engine-console/validator-engine-console-query.cpp

@@ -26,9 +26,12 @@
     Copyright 2017-2020 Telegram Systems LLP
 */
 #include "validator-engine-console-query.h"
+#include "auto/tl/ton_api.h"
+#include "td/utils/StringBuilder.h"
 #include "validator-engine-console.h"
 #include "terminal/terminal.h"
 #include "td/utils/filesystem.h"
+#include "overlay/overlays.h"
 
 #include <cctype>
 
@@ -720,3 +723,193 @@ td::Status CheckDhtServersQuery::receive(td::BufferSlice data) {
   }
   return td::Status::OK();
 }
+
+td::Status SignCertificateQuery::run() {
+  TRY_RESULT_ASSIGN(overlay_, tokenizer_.get_token<td::Bits256>());
+  TRY_RESULT_ASSIGN(id_, tokenizer_.get_token<td::Bits256>());
+  TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token<td::int32>());
+  TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token<td::uint32>());
+  TRY_RESULT_ASSIGN(signer_, tokenizer_.get_token<ton::PublicKeyHash>());
+  TRY_RESULT_ASSIGN(out_file_, tokenizer_.get_token<std::string>());
+  return td::Status::OK();
+}
+
+td::Status SignCertificateQuery::send() {
+  auto cid = ton::create_serialize_tl_object<ton::ton_api::overlay_certificateId>(overlay_, id_, expire_at_, max_size_);
+  auto sign = ton::create_serialize_tl_object<ton::ton_api::engine_validator_sign>(signer_.tl(), std::move(cid));
+  auto pub = ton::create_serialize_tl_object<ton::ton_api::engine_validator_exportPublicKey>(signer_.tl());
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(pub),
+      td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+        if (R.is_error()) {
+          td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error());
+        } else {
+          td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok());
+        }
+      }));
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(sign),
+      td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+        if (R.is_error()) {
+          td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error());
+        } else {
+          td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok());
+        }
+      }));
+  return td::Status::OK();
+}
+
+void SignCertificateQuery::receive_pubkey(td::BufferSlice R) {
+  auto f = ton::fetch_tl_object<ton::ton_api::PublicKey>(R.as_slice(), true);
+  if (f.is_error()) {
+    handle_error(f.move_as_error_prefix("Failed to get pubkey: "));
+    return;
+  }
+  pubkey_ = f.move_as_ok();
+  has_pubkey_ = true;
+  if(has_signature_) {
+    save_certificate();
+  }
+}
+
+
+td::Status SignCertificateQuery::receive(td::BufferSlice data) {
+  UNREACHABLE();
+}
+
+void SignCertificateQuery::receive_signature(td::BufferSlice R) {
+  auto f = ton::fetch_tl_object<ton::ton_api::engine_validator_signature>(R.as_slice(), true);
+  if(f.is_error()){
+    handle_error(f.move_as_error_prefix("Failed to get signature: "));
+    return;
+  }
+  signature_ = std::move(f.move_as_ok()->signature_);
+  if(has_pubkey_) {
+    save_certificate();
+  }
+}
+
+void SignCertificateQuery::save_certificate() {
+  auto c = ton::create_serialize_tl_object<ton::ton_api::overlay_certificate>(
+        std::move(pubkey_), expire_at_, max_size_, std::move(signature_));
+  auto w = td::write_file(out_file_, c.as_slice());
+  if(w.is_error()) {
+    handle_error(w.move_as_error_prefix("Failed to write certificate to file: "));
+    return;
+  }
+  td::TerminalIO::out() << "saved certificate\n";
+  stop();
+}
+
+td::Status ImportCertificateQuery::run() {
+  TRY_RESULT_ASSIGN(overlay_, tokenizer_.get_token<td::Bits256>());
+  TRY_RESULT_ASSIGN(id_, tokenizer_.get_token<td::Bits256>());
+  TRY_RESULT_ASSIGN(kh_, tokenizer_.get_token<ton::PublicKeyHash>());
+  TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token<std::string>());
+  return td::Status::OK();
+}
+
+td::Status ImportCertificateQuery::send() {
+  TRY_RESULT(data, td::read_file(in_file_));
+  TRY_RESULT_PREFIX(cert, ton::fetch_tl_object<ton::ton_api::overlay_Certificate>(data.as_slice(), true),
+                    "incorrect certificate");
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_importCertificate>(
+                overlay_,
+                ton::create_tl_object<ton::ton_api::adnl_id_short>(id_),
+                ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(kh_.tl()),
+                std::move(cert)
+           );
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+td::Status GetOverlaysStatsQuery::run() {
+  return td::Status::OK();
+}
+
+td::Status GetOverlaysStatsQuery::send() {
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_getOverlaysStats>();
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_overlaysStats>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  for (auto &s : f->overlays_) {
+    td::StringBuilder sb;
+    sb << "overlay_id=" << s->overlay_id_ << " adnl_id=" << s->adnl_id_ << "\n";
+    sb << "  nodes:";
+    for (auto &n : s->nodes_) {
+      sb << "   " << n->id_ << "\n";
+    }
+    sb << "  stats:\n";
+    for (auto &t : s->stats_) {
+      sb << "    " << t->key_ << "\t" << t->value_ << "\n";
+    }
+    td::TerminalIO::output(sb.as_cslice());
+  }
+  return td::Status::OK();
+}
+
+
+td::Status ImportCertificateQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  td::TerminalIO::out() << "successfully sent certificate to overlay manager\n";
+  return td::Status::OK();
+}
+
+
+td::Status SignShardOverlayCertificateQuery::run() {
+  TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token<td::int32>());
+  TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>() );
+  TRY_RESULT_ASSIGN(key_, tokenizer_.get_token<ton::PublicKeyHash>());
+  TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token<td::int32>());
+  TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token<td::uint32>());
+  TRY_RESULT_ASSIGN(out_file_, tokenizer_.get_token<std::string>());
+
+  return td::Status::OK();
+}
+
+td::Status SignShardOverlayCertificateQuery::send() {
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_signShardOverlayCertificate>
+             (wc_, shard_, ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(key_.tl()), expire_at_, max_size_);
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(c, ton::fetch_tl_object<ton::ton_api::overlay_certificate>(data.as_slice(), true),
+                    "received incorrect cert: ");
+  auto w = td::write_file(out_file_, data.as_slice());
+  if(w.is_error()) {
+    return w.move_as_error_prefix("Failed to write certificate to file: ");
+  }
+  td::TerminalIO::out() << "saved certificate\n";
+
+  return td::Status::OK();
+}
+
+td::Status ImportShardOverlayCertificateQuery::run() {
+  TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token<td::int32>());
+  TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>() );
+  TRY_RESULT_ASSIGN(key_, tokenizer_.get_token<ton::PublicKeyHash>());
+  TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token<std::string>());
+
+  return td::Status::OK();
+}
+
+td::Status ImportShardOverlayCertificateQuery::send() {
+  TRY_RESULT(data, td::read_file(in_file_));
+  TRY_RESULT_PREFIX(cert, ton::fetch_tl_object<ton::ton_api::overlay_Certificate>(data.as_slice(), true),
+                    "incorrect certificate");
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_importShardOverlayCertificate>
+             (wc_, shard_, ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(key_.tl()), std::move(cert));
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  td::TerminalIO::out() << "successfully sent certificate to overlay manager\n";
+  return td::Status::OK();
+}

validator-engine-console/validator-engine-console-query.h

@@ -903,3 +903,139 @@ class CheckDhtServersQuery : public Query {
  private:
   ton::PublicKeyHash id_;
 };
+
+class GetOverlaysStatsQuery : public Query {
+ public:
+  GetOverlaysStatsQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "getoverlaysstats";
+  }
+  static std::string get_help() {
+    return "getoverlaysstats\tgets stats for all overlays";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+};
+
+class SignCertificateQuery : public Query {
+ public:
+  SignCertificateQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "signcert";
+  }
+  static std::string get_help() {
+    return "signcert <overlayid> <adnlid> <expireat> <maxsize> <signwith> <outfile>\tsign overlay certificate by <signwith> key";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+  void receive_pubkey(td::BufferSlice R);
+  void receive_signature(td::BufferSlice R);
+
+
+ private:
+   void save_certificate();
+
+  td::Bits256 overlay_;
+  td::Bits256 id_;
+  td::int32 expire_at_;
+  td::uint32 max_size_;
+  std::string out_file_;
+  ton::PublicKeyHash signer_;
+  td::BufferSlice signature_;
+  std::unique_ptr<ton::ton_api::PublicKey> pubkey_;
+  bool has_signature_{0};
+  bool has_pubkey_{0};
+};
+
+class ImportCertificateQuery : public Query {
+ public:
+  ImportCertificateQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "importcert";
+  }
+  static std::string get_help() {
+    return "importcert <overlayid> <adnlid> <key> <certfile>\timport overlay certificate for specific key";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+
+ private:
+  td::Bits256 overlay_;
+  td::Bits256 id_;
+  ton::PublicKeyHash kh_;
+  std::string in_file_;
+};
+
+class SignShardOverlayCertificateQuery : public Query {
+ public:
+  SignShardOverlayCertificateQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "signshardoverlaycert";
+  }
+  static std::string get_help() {
+    return "signshardoverlaycert <workchain> <shardprefix> <key> <expireat> <maxsize> <outfile>\tsign certificate for <key> in currently active shard overlay";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+
+ private:
+
+  td::int32 wc_;
+  td::int64 shard_;
+  td::int32 expire_at_;
+  ton::PublicKeyHash key_;
+  td::uint32 max_size_;
+  std::string out_file_;
+};
+
+
+class ImportShardOverlayCertificateQuery : public Query {
+ public:
+  ImportShardOverlayCertificateQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "importshardoverlaycert";
+  }
+  static std::string get_help() {
+    return "importshardoverlaycert <workchain> <shardprefix> <key> <certfile>\timport certificate for <key> in currently active shard overlay";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+
+ private:
+
+  td::int32 wc_;
+  td::int64 shard_;
+  ton::PublicKeyHash key_;
+  std::string in_file_;
+};
+

validator-engine-console/validator-engine-console.cpp

@@ -134,6 +134,11 @@ void ValidatorEngineConsole::run() {
   add_query_runner(std::make_unique<QueryRunnerImpl<CreateProposalVoteQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<CreateComplaintVoteQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<CheckDhtServersQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<SignCertificateQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<ImportCertificateQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<GetOverlaysStatsQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<ImportShardOverlayCertificateQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<SignShardOverlayCertificateQuery>>());
 }
 
 bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise<td::BufferSlice> promise) {
@@ -258,7 +263,8 @@ int main(int argc, char* argv[]) {
     std::exit(2);
   });
   p.add_option('V', "version", "shows validator-engine-console build information", [&]() {
-    std::cout << "validator-engine-console build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n";
+    std::cout << "validator-engine-console build information: [ Commit: " << GitMetadata::CommitSHA1()
+              << ", Date: " << GitMetadata::CommitDate() << "]\n";
     std::exit(0);
   });
   p.add_checked_option('a', "address", "server address", [&](td::Slice arg) {

validator-engine/validator-engine.cpp

@@ -27,6 +27,11 @@
 */
 #include "validator-engine.hpp"
 
+#include "auto/tl/ton_api.h"
+#include "overlay-manager.h"
+#include "td/actor/actor.h"
+#include "tl-utils/tl-utils.hpp"
+#include "tl/TlObject.h"
 #include "ton/ton-types.h"
 #include "ton/ton-tl.hpp"
 #include "ton/ton-io.hpp"
@@ -1309,6 +1314,9 @@ td::Status ValidatorEngine::load_global_config() {
   if (state_ttl_ != 0) {
     validator_options_.write().set_state_ttl(state_ttl_);
   }
+  if (max_mempool_num_ != 0) {
+    validator_options_.write().set_max_mempool_num(max_mempool_num_);
+  }
   if (block_ttl_ != 0) {
     validator_options_.write().set_block_ttl(block_ttl_);
   }
@@ -3140,6 +3148,135 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_createCom
       .release();
 }
 
+void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCertificate &query, td::BufferSlice data,
+                                        ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise) {
+  if (!(perm & ValidatorEnginePermissions::vep_modify)) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
+    return;
+  }
+  if (keyring_.empty()) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started")));
+    return;
+  }
+
+  if (!started_) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started")));
+    return;
+  }
+  auto r = ton::overlay::Certificate::create(std::move(query.cert_));
+  if(r.is_error()) {
+    promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: ")));
+  }
+  //TODO force Overlays::update_certificate to return result
+  /*auto P = td::PromiseCreator::lambda(
+      [promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+        if (R.is_error()) {
+          promise.set_value(create_control_query_error(R.move_as_error()));
+        } else {
+          promise.set_value(
+            ton::serialize_tl_object(ton::create_tl_object<ton::ton_api::engine_validator_success>(), true));
+        }
+      });
+  */
+  td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::update_certificate,
+                            ton::adnl::AdnlNodeIdShort{query.local_id_->id_},
+                            ton::overlay::OverlayIdShort{query.overlay_id_},
+                            ton::PublicKeyHash{query.signed_key_->key_hash_},
+                            r.move_as_ok());
+  promise.set_value(
+            ton::serialize_tl_object(ton::create_tl_object<ton::ton_api::engine_validator_success>(), true)
+  );
+}
+
+void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data,
+                                        ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise) {
+  if (!(perm & ValidatorEnginePermissions::vep_modify)) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
+    return;
+  }
+  if (keyring_.empty()) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started")));
+    return;
+  }
+
+  if (!started_) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started")));
+    return;
+  }
+  auto r = ton::overlay::Certificate::create(std::move(query.cert_));
+  if(r.is_error()) {
+    promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: ")));
+  }
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+    if (R.is_error()) {
+      promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to import cert: ")));
+    } else {
+      promise.set_value(
+          ton::serialize_tl_object(ton::create_tl_object<ton::ton_api::engine_validator_success>(), true));
+    }
+  });
+  ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast<ton::ShardId>(query.shard_)};
+  td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate,
+                            shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P));
+}
+
+void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data,
+                                        ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise) {
+  if (!(perm & ValidatorEnginePermissions::vep_modify)) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
+    return;
+  }
+  if (keyring_.empty()) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started")));
+    return;
+  }
+
+  if (!started_) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started")));
+    return;
+  }
+  ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast<ton::ShardId>(query.shard_)};
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+    if (R.is_error()) {
+      promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to import cert: ")));
+    } else {
+      promise.set_value(R.move_as_ok());
+    }
+  });
+  td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate,
+                            shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(P));
+}
+
+
+void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data,
+                                        ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise) {
+  if (!(perm & ValidatorEnginePermissions::vep_default)) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
+    return;
+  }
+
+  if (keyring_.empty()) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started")));
+    return;
+  }
+
+  if (!started_) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started")));
+    return;
+  }
+
+  td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::get_stats,
+                          [promise = std::move(promise)](
+                              td::Result<ton::tl_object_ptr<ton::ton_api::engine_validator_overlaysStats>> R) mutable {
+                            if (R.is_ok()) {
+                              promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true));
+                            } else {
+                              promise.set_value(create_control_query_error(
+                                  td::Status::Error(ton::ErrorCode::notready, "overlay manager not ready")));
+                            }
+                          });
+}
+
 void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src,
                                             ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data,
                                             td::Promise<td::BufferSlice> promise) {
@@ -3291,7 +3428,8 @@ int main(int argc, char *argv[]) {
     SET_VERBOSITY_LEVEL(v);
   });
   p.add_option('V', "version", "shows validator-engine build information", [&]() {
-    std::cout << "validator-engine build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n";
+    std::cout << "validator-engine build information: [ Commit: " << GitMetadata::CommitSHA1()
+              << ", Date: " << GitMetadata::CommitDate() << "]\n";
     std::exit(0);
   });
   p.add_option('h', "help", "prints_help", [&]() {
@@ -3336,6 +3474,10 @@ int main(int argc, char *argv[]) {
     auto v = td::to_double(fname);
     acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); });
   });
+  p.add_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) {
+    auto v = td::to_double(fname);
+    acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); });
+  });
   p.add_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400",
                [&](td::Slice fname) {
                  auto v = td::to_double(fname);
@@ -3367,8 +3509,8 @@ int main(int argc, char *argv[]) {
         acts.push_back([&x, seq]() { td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain, seq); });
         return td::Status::OK();
       });
-  p.add_checked_option(
+  p.add_checked_option('F', "unsafe-catchain-rotate", "use forceful and DANGEROUS catchain rotation",
-      'F', "unsafe-catchain-rotate", "use forceful and DANGEROUS catchain rotation", [&](td::Slice params) {
+                       [&](td::Slice params) {
                          auto pos1 = params.find(':');
                          TRY_RESULT(b_seq, td::to_integer_safe<ton::BlockSeqno>(params.substr(0, pos1)));
                          params = params.substr(++pos1, params.size());
@@ -3376,7 +3518,9 @@ int main(int argc, char *argv[]) {
                          TRY_RESULT(cc_seq, td::to_integer_safe<ton::CatchainSeqno>(params.substr(0, pos2)));
                          params = params.substr(++pos2, params.size());
                          auto h = std::stoi(params.substr(0, params.size()).str());
-        acts.push_back([&x, b_seq, cc_seq, h]() { td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain_rotation, b_seq, cc_seq, h); });
+                         acts.push_back([&x, b_seq, cc_seq, h]() {
+                           td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain_rotation, b_seq, cc_seq, h);
+                         });
                          return td::Status::OK();
                        });
   td::uint32 threads = 7;

validator-engine/validator-engine.hpp

@@ -28,6 +28,7 @@
 #pragma once
 
 #include "adnl/adnl.h"
+#include "auto/tl/ton_api.h"
 #include "rldp/rldp.h"
 #include "dht/dht.h"
 #include "validator/manager.h"
@@ -191,6 +192,7 @@ class ValidatorEngine : public td::actor::Actor {
   std::map<CI_key, td::uint32> control_permissions_;
 
   double state_ttl_ = 0;
+  double max_mempool_num_ = 0;
   double block_ttl_ = 0;
   double sync_ttl_ = 0;
   double archive_ttl_ = 0;
@@ -223,6 +225,9 @@ class ValidatorEngine : public td::actor::Actor {
   void set_state_ttl(double t) {
     state_ttl_ = t;
   }
+  void set_max_mempool_num(double t) {
+    max_mempool_num_ = t;
+  }
   void set_block_ttl(double t) {
     block_ttl_ = t;
   }
@@ -391,6 +396,14 @@ class ValidatorEngine : public td::actor::Actor {
                          ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
   void run_control_query(ton::ton_api::engine_validator_createComplaintVote &query, td::BufferSlice data,
                          ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
+  void run_control_query(ton::ton_api::engine_validator_importCertificate &query, td::BufferSlice data,
+                         ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
+  void run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data,
+                         ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
+  void run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data,
+                         ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
+  void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data,
+                         ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
   template <class T>
   void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
                          td::Promise<td::BufferSlice> promise) {

validator/db/archive-manager.cpp

@@ -336,6 +336,28 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t
 
 void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data,
                                           td::Promise<td::Unit> promise) {
+  auto create_writer = [&](std::string path, td::Promise<std::string> P) {
+    td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/",
+                                           std::move(path), std::move(data), std::move(P))
+        .release();
+  };
+  add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer));
+}
+
+void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                              std::function<td::Status(td::FileFd&)> write_state,
+                                              td::Promise<td::Unit> promise) {
+  auto create_writer = [&](std::string path, td::Promise<std::string> P) {
+    td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/",
+                                           std::move(path), std::move(write_state), std::move(P))
+        .release();
+  };
+  add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer));
+}
+
+void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                               td::Promise<td::Unit> promise,
+                                               std::function<void(std::string, td::Promise<std::string>)> create_writer) {
   auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}};
   auto hash = id.hash();
   if (perm_states_.find(hash) != perm_states_.end()) {
@@ -353,8 +375,7 @@ void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt master
           promise.set_value(td::Unit());
         }
       });
-  td::actor::create_actor<db::WriteFile>("writefile", db_root_ + "/archive/tmp/", path, std::move(data), std::move(P))
+  create_writer(std::move(path), std::move(P));
-      .release();
 }
 
 void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) {

validator/db/archive-manager.hpp

@@ -45,6 +45,9 @@ class ArchiveManager : public td::actor::Actor {
   void add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise<td::Unit> promise);
   void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data,
                             td::Promise<td::Unit> promise);
+  void add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                std::function<td::Status(td::FileFd&)> write_state,
+                                td::Promise<td::Unit> promise);
   void get_zero_state(BlockIdExt block_id, td::Promise<td::BufferSlice> promise);
   void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::BufferSlice> promise);
   void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,
@@ -137,6 +140,8 @@ class ArchiveManager : public td::actor::Actor {
   PackageId get_max_temp_file_desc_idx();
   PackageId get_prev_temp_file_desc_idx(PackageId id);
 
+  void add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise,
+                                 std::function<void(std::string, td::Promise<std::string>)> create_writer);
   void written_perm_state(FileReferenceShort id);
 
   void persistent_state_gc(FileHash last);

validator/db/files-async.hpp

@@ -52,30 +52,50 @@ class WriteFile : public td::actor::Actor {
     auto res = R.move_as_ok();
     auto file = std::move(res.first);
     auto old_name = res.second;
-    td::uint64 offset = 0;
+    auto status = write_data_(file);
-    while (data_.size() > 0) {
+    if (!status.is_error()) {
-      auto R = file.pwrite(data_.as_slice(), offset);
+      status = file.sync();
-      auto s = R.move_as_ok();
+    }
-      offset += s;
+    if (status.is_error()) {
-      data_.confirm_read(s);
+      td::unlink(old_name);
+      promise_.set_error(std::move(status));
+      stop();
+      return;
     }
-    file.sync().ensure();
     if (new_name_.length() > 0) {
-      td::rename(old_name, new_name_).ensure();
+      status = td::rename(old_name, new_name_);
+      if (status.is_error()) {
+        promise_.set_error(std::move(status));
+      } else {
         promise_.set_value(std::move(new_name_));
+      }
     } else {
       promise_.set_value(std::move(old_name));
     }
     stop();
   }
+  WriteFile(std::string tmp_dir, std::string new_name, std::function<td::Status(td::FileFd&)> write_data,
+            td::Promise<std::string> promise)
+      : tmp_dir_(tmp_dir), new_name_(new_name), write_data_(std::move(write_data)), promise_(std::move(promise)) {
+  }
   WriteFile(std::string tmp_dir, std::string new_name, td::BufferSlice data, td::Promise<std::string> promise)
-      : tmp_dir_(tmp_dir), new_name_(new_name), data_(std::move(data)), promise_(std::move(promise)) {
+      : tmp_dir_(tmp_dir), new_name_(new_name), promise_(std::move(promise)) {
+    write_data_ = [data_ptr = std::make_shared<td::BufferSlice>(std::move(data))] (td::FileFd& fd) {
+      auto data = std::move(*data_ptr);
+      td::uint64 offset = 0;
+      while (data.size() > 0) {
+        TRY_RESULT(s, fd.pwrite(data.as_slice(), offset));
+        offset += s;
+        data.confirm_read(s);
+      }
+      return td::Status::OK();
+    };
   }
 
  private:
   const std::string tmp_dir_;
   std::string new_name_;
-  td::BufferSlice data_;
+  std::function<td::Status(td::FileFd&)> write_data_;
   td::Promise<std::string> promise_;
 };
 

validator/db/rootdb.cpp

@@ -276,6 +276,13 @@ void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterc
                           std::move(state), std::move(promise));
 }
 
+void RootDb::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                             std::function<td::Status(td::FileFd&)> write_data,
+                                             td::Promise<td::Unit> promise) {
+  td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state_gen, block_id, masterchain_block_id,
+                          std::move(write_data), std::move(promise));
+}
+
 void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id,
                                        td::Promise<td::BufferSlice> promise) {
   td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id,

validator/db/rootdb.hpp

@@ -69,6 +69,9 @@ class RootDb : public Db {
 
   void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
                                    td::Promise<td::Unit> promise) override;
+  void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                       std::function<td::Status(td::FileFd&)> write_data,
+                                       td::Promise<td::Unit> promise) override;
   void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id,
                                  td::Promise<td::BufferSlice> promise) override;
   void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,

validator/fabric.h

@@ -46,6 +46,8 @@ td::Result<std::vector<td::Ref<ShardTopBlockDescription>>> create_new_shard_bloc
 
 td::Ref<BlockSignatureSet> create_signature_set(std::vector<BlockSignature> sig_set);
 
+void run_check_external_message(td::BufferSlice data, td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise);
+
 void run_accept_block_query(BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
                             td::Ref<ValidatorSet> validator_set, td::Ref<BlockSignatureSet> signatures,
                             td::Ref<BlockSignatureSet> approve_signatures, bool send_broadcast,
@@ -81,6 +83,8 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b
                           td::Promise<BlockCandidate> promise);
 void run_liteserver_query(td::BufferSlice data, td::actor::ActorId<ValidatorManager> manager,
                           td::actor::ActorId<LiteServerCache> cache, td::Promise<td::BufferSlice> promise);
+void run_fetch_account_state(WorkchainId wc, StdSmcAddress  addr, td::actor::ActorId<ValidatorManager> manager,
+                             td::Promise<std::tuple<td::Ref<vm::CellSlice>,UnixTime,LogicalTime,std::unique_ptr<block::ConfigInfo>>> promise);
 void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block,
                                           td::Ref<MasterchainState> masterchain_state,
                                           td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,

validator/full-node-shard.cpp

@@ -16,6 +16,8 @@
 
     Copyright 2017-2020 Telegram Systems LLP
 */
+#include "auto/tl/ton_api.h"
+#include "overlays.h"
 #include "td/utils/SharedSlice.h"
 #include "full-node-shard.hpp"
 #include "full-node-shard-queries.hpp"
@@ -79,6 +81,10 @@ void FullNodeShardImpl::create_overlay() {
     void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override {
       td::actor::send_closure(node_, &FullNodeShardImpl::receive_broadcast, src, std::move(data));
     }
+    void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data,
+                         td::Promise<td::Unit> promise) override {
+      td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise));
+    }
     Callback(td::actor::ActorId<FullNodeShardImpl> node) : node_(node) {
     }
 
@@ -95,9 +101,21 @@ void FullNodeShardImpl::create_overlay() {
   }
 }
 
+void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise<td::Unit> promise) {
+  auto B = fetch_tl_object<ton_api::tonNode_externalMessageBroadcast>(std::move(broadcast), true);
+  if (B.is_error()) {
+    return promise.set_error(B.move_as_error_prefix("failed to parse external message broadcast: "));
+  }
+
+  auto q = B.move_as_ok();
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_external_message,
+                          std::move(q->message_->data_), std::move(promise));
+}
+
 void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise<td::Unit> promise) {
   td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_);
   adnl_id_ = adnl_id;
+  local_id_ = adnl_id_.pubkey_hash();
   create_overlay();
 }
 
@@ -804,8 +822,9 @@ void FullNodeShardImpl::sign_new_certificate(PublicKeyHash sign_by) {
     return;
   }
 
-  ton::overlay::Certificate cert{sign_by, static_cast<td::int32>(td::Clocks::system() + 3600),
+  ton::overlay::Certificate cert{
-                                 overlay::Overlays::max_fec_broadcast_size(), td::BufferSlice{}};
+      sign_by, static_cast<td::int32>(td::Clocks::system() + 3600), overlay::Overlays::max_fec_broadcast_size(),
+      overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, td::BufferSlice{}};
   auto to_sign = cert.to_sign(overlay_id_, local_id_);
 
   auto P = td::PromiseCreator::lambda(
@@ -830,6 +849,38 @@ void FullNodeShardImpl::signed_new_certificate(ton::overlay::Certificate cert) {
   td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_);
 }
 
+void FullNodeShardImpl::sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expire_at, td::uint32 max_size, td::Promise<td::BufferSlice> promise) {
+  auto sign_by = sign_cert_by_;
+  if (sign_by.is_zero()) {
+    promise.set_error(td::Status::Error("Node has no key with signing authority"));
+    return;
+  }
+
+  ton::overlay::Certificate cert{
+      sign_by, static_cast<td::int32>(expire_at), max_size,
+      overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, td::BufferSlice{}};
+  auto to_sign = cert.to_sign(overlay_id_, signed_key);
+
+  auto P = td::PromiseCreator::lambda(
+      [SelfId = actor_id(this), expire_at = expire_at, max_size = max_size, promise = std::move(promise)](td::Result<std::pair<td::BufferSlice, PublicKey>> R) mutable {
+        if (R.is_error()) {
+          promise.set_error(R.move_as_error_prefix("failed to create certificate: failed to sign: "));
+        } else {
+          auto p = R.move_as_ok();
+          auto c = ton::create_serialize_tl_object<ton::ton_api::overlay_certificate>(p.second.tl(), static_cast<td::int32>(expire_at), max_size, std::move(p.first));
+          promise.set_value(std::move(c));
+        }
+      });
+  td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_add_get_public_key, sign_by, std::move(to_sign),
+                          std::move(P));
+}
+
+void FullNodeShardImpl::import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr<ton::overlay::Certificate> cert, td::Promise<td::Unit> promise) {
+  td::actor::send_closure(overlays_, &ton::overlay::Overlays::update_certificate,
+                                     adnl_id_, overlay_id_, signed_key, cert);
+  promise.set_value( td::Unit()  );
+}
+
 void FullNodeShardImpl::update_validators(std::vector<PublicKeyHash> public_key_hashes, PublicKeyHash local_hash) {
   if (!client_.empty()) {
     return;
@@ -845,7 +896,7 @@ void FullNodeShardImpl::update_validators(std::vector<PublicKeyHash> public_key_
     authorized_keys.emplace(key, overlay::Overlays::max_fec_broadcast_size());
   }
 
-  rules_ = overlay::OverlayPrivacyRules{1 << 14, std::move(authorized_keys)};
+  rules_ = overlay::OverlayPrivacyRules{overlay::Overlays::max_fec_broadcast_size(), overlay::CertificateFlags::AllowFec, std::move(authorized_keys)};
   td::actor::send_closure(overlays_, &overlay::Overlays::set_privacy_rules, adnl_id_, overlay_id_, rules_);
 
   if (update_cert) {
@@ -949,8 +1000,7 @@ void FullNodeShardImpl::update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, do
   }
 }
 
-void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t,
+void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data) {
-                                                   td::BufferSlice data) {
   auto it = neighbours_.find(adnl_id);
   if (it == neighbours_.end()) {
     return;

validator/full-node-shard.h

@@ -42,6 +42,10 @@ class FullNodeShard : public td::actor::Actor {
   virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0;
   virtual void send_broadcast(BlockBroadcast broadcast) = 0;
 
+  virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise<td::BufferSlice> promise) = 0;
+  virtual void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr<ton::overlay::Certificate> cert, td::Promise<td::Unit> promise) = 0;
+
+
   virtual void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout,
                               td::Promise<ReceivedBlock> promise) = 0;
   virtual void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout,

validator/full-node-shard.hpp

@@ -19,6 +19,8 @@
 #pragma once
 
 #include "full-node-shard.h"
+#include "td/actor/PromiseFuture.h"
+#include "td/utils/port/Poll.h"
 
 namespace ton {
 
@@ -139,6 +141,7 @@ class FullNodeShardImpl : public FullNodeShard {
   void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query);
   void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query);
   void receive_broadcast(PublicKeyHash src, td::BufferSlice query);
+  void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise<td::Unit> promise);
 
   void send_ihr_message(td::BufferSlice data) override;
   void send_external_message(td::BufferSlice data) override;
@@ -167,6 +170,10 @@ class FullNodeShardImpl : public FullNodeShard {
   void alarm() override;
 
   void update_validators(std::vector<PublicKeyHash> public_key_hashes, PublicKeyHash local_hash) override;
+
+  void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise<td::BufferSlice> promise) override;
+  void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr<ton::overlay::Certificate> cert, td::Promise<td::Unit> promise) override;
+
   void sign_new_certificate(PublicKeyHash sign_by);
   void signed_new_certificate(ton::overlay::Certificate cert);
 

validator/full-node.cpp

@@ -76,6 +76,27 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise<td::Unit> pr
   promise.set_value(td::Unit());
 }
 
+void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key,
+                                                  td::uint32 expiry_at, td::uint32 max_size,
+                                                  td::Promise<td::BufferSlice> promise) {
+    auto it = shards_.find(shard_id);
+    if(it == shards_.end()) {
+      promise.set_error(td::Status::Error(ErrorCode::error, "shard not found"));
+      return;
+    }
+    td::actor::send_closure(it->second, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, std::move(promise));
+}
+
+void FullNodeImpl::import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key,
+                                                    std::shared_ptr<ton::overlay::Certificate> cert,
+                                                    td::Promise<td::Unit> promise) {
+    auto it = shards_.find(shard_id);
+    if(it == shards_.end()) {
+      promise.set_error(td::Status::Error(ErrorCode::error, "shard not found"));
+    }
+    td::actor::send_closure(it->second, &FullNodeShard::import_overlay_certificate, signed_key, cert, std::move(promise));
+}
+
 void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise<td::Unit> promise) {
   adnl_id_ = adnl_id;
 
@@ -86,6 +107,7 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise<td:
   for (auto &s : shards_) {
     td::actor::send_closure(s.second, &FullNodeShard::update_adnl_id, adnl_id, ig.get_promise());
   }
+  local_id_ = adnl_id_.pubkey_hash();
 }
 
 void FullNodeImpl::initial_read_complete(BlockHandle top_handle) {
@@ -345,10 +367,14 @@ void FullNodeImpl::new_key_block(BlockHandle handle) {
 
 void FullNodeImpl::start_up() {
   if (local_id_.is_zero()) {
+    if(adnl_id_.is_zero()) {
       auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()};
       local_id_ = pk.compute_short_id();
 
       td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(pk), true, [](td::Unit) {});
+    } else {
+      local_id_ = adnl_id_.pubkey_hash();
+    }
   }
   class Callback : public ValidatorManagerInterface::Callback {
    public:

validator/full-node.h

@@ -53,6 +53,13 @@ class FullNode : public td::actor::Actor {
   virtual void add_permanent_key(PublicKeyHash key, td::Promise<td::Unit> promise) = 0;
   virtual void del_permanent_key(PublicKeyHash key, td::Promise<td::Unit> promise) = 0;
 
+  virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key,
+                                              td::uint32 expiry_at, td::uint32 max_size,
+                                              td::Promise<td::BufferSlice> promise) = 0;
+  virtual void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key,
+                                                std::shared_ptr<ton::overlay::Certificate> cert,
+                                                td::Promise<td::Unit> promise) = 0;
+
   virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise<td::Unit> promise) = 0;
 
   static constexpr td::uint32 max_block_size() {

validator/full-node.hpp

@@ -42,6 +42,14 @@ class FullNodeImpl : public FullNode {
   void add_permanent_key(PublicKeyHash key, td::Promise<td::Unit> promise) override;
   void del_permanent_key(PublicKeyHash key, td::Promise<td::Unit> promise) override;
 
+  void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key,
+                                      td::uint32 expiry_at, td::uint32 max_size,
+                                      td::Promise<td::BufferSlice> promise) override;
+  void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key,
+                                        std::shared_ptr<ton::overlay::Certificate> cert,
+                                        td::Promise<td::Unit> promise) override;
+
+
   void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise<td::Unit> promise) override;
 
   void add_shard(ShardIdFull shard);

validator/impl/collator-impl.h

@@ -103,6 +103,27 @@ class Collator final : public td::actor::Actor {
     return 2;
   }
 
+  static td::Result<std::unique_ptr<block::ConfigInfo>>
+                     impl_fetch_config_params(std::unique_ptr<block::ConfigInfo> config,
+                                              Ref<vm::Cell>* old_mparams,
+                                              std::vector<block::StoragePrices>* storage_prices,
+                                              block::StoragePhaseConfig* storage_phase_cfg,
+                                              td::BitArray<256>* rand_seed,
+                                              block::ComputePhaseConfig* compute_phase_cfg,
+                                              block::ActionPhaseConfig* action_phase_cfg,
+                                              td::RefInt256* masterchain_create_fee,
+                                              td::RefInt256* basechain_create_fee,
+                                              WorkchainId wc);
+
+  static td::Result<std::unique_ptr<block::Transaction>>
+                        impl_create_ordinary_transaction(Ref<vm::Cell> msg_root,
+                                                         block::Account* acc,
+                                                         UnixTime utime, LogicalTime lt,
+                                                         block::StoragePhaseConfig* storage_phase_cfg,
+                                                         block::ComputePhaseConfig* compute_phase_cfg,
+                                                         block::ActionPhaseConfig* action_phase_cfg,
+                                                         bool external, LogicalTime after_lt);
+
  private:
   void start_up() override;
   void alarm() override;

validator/impl/collator.cpp

@@ -1556,68 +1556,92 @@ bool Collator::init_lt() {
 }
 
 bool Collator::fetch_config_params() {
-  old_mparams_ = config_->get_config_param(9);
+  auto res = impl_fetch_config_params(std::move(config_),
-  {
+                                      &old_mparams_, &storage_prices_, &storage_phase_cfg_,
-    auto res = config_->get_storage_prices();
+                                      &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_,
+                                      &masterchain_create_fee_, &basechain_create_fee_,
+                                      workchain()
+                                     );
   if (res.is_error()) {
       return fatal_error(res.move_as_error());
   }
-    storage_prices_ = res.move_as_ok();
+  config_ = res.move_as_ok();
+  return true;
+}
+
+td::Result<std::unique_ptr<block::ConfigInfo>>
+           Collator::impl_fetch_config_params(std::unique_ptr<block::ConfigInfo> config,
+                                              Ref<vm::Cell>* old_mparams,
+                                              std::vector<block::StoragePrices>* storage_prices,
+                                              block::StoragePhaseConfig* storage_phase_cfg,
+                                              td::BitArray<256>* rand_seed,
+                                              block::ComputePhaseConfig* compute_phase_cfg,
+                                              block::ActionPhaseConfig* action_phase_cfg,
+                                              td::RefInt256* masterchain_create_fee,
+                                              td::RefInt256* basechain_create_fee,
+                                              WorkchainId wc) {
+  *old_mparams = config->get_config_param(9);
+  {
+    auto res = config->get_storage_prices();
+    if (res.is_error()) {
+      return res.move_as_error();
+    }
+    *storage_prices = res.move_as_ok();
   }
   {
     // generate rand seed
-    prng::rand_gen().strong_rand_bytes(rand_seed_.data(), 32);
+    prng::rand_gen().strong_rand_bytes(rand_seed->data(), 32);
-    LOG(DEBUG) << "block random seed set to " << rand_seed_.to_hex();
+    LOG(DEBUG) << "block random seed set to " << rand_seed->to_hex();
   }
   {
     // compute compute_phase_cfg / storage_phase_cfg
-    auto cell = config_->get_config_param(is_masterchain() ? 20 : 21);
+    auto cell = config->get_config_param(wc == ton::masterchainId ? 20 : 21);
     if (cell.is_null()) {
-      return fatal_error("cannot fetch current gas prices and limits from masterchain configuration");
+      return td::Status::Error(-668, "cannot fetch current gas prices and limits from masterchain configuration");
     }
-    if (!compute_phase_cfg_.parse_GasLimitsPrices(std::move(cell), storage_phase_cfg_.freeze_due_limit,
+    if (!compute_phase_cfg->parse_GasLimitsPrices(std::move(cell), storage_phase_cfg->freeze_due_limit,
-                                                  storage_phase_cfg_.delete_due_limit)) {
+                                                  storage_phase_cfg->delete_due_limit)) {
-      return fatal_error("cannot unpack current gas prices and limits from masterchain configuration");
+      return td::Status::Error(-668, "cannot unpack current gas prices and limits from masterchain configuration");
     }
-    compute_phase_cfg_.block_rand_seed = rand_seed_;
+    compute_phase_cfg->block_rand_seed = *rand_seed;
-    compute_phase_cfg_.libraries = std::make_unique<vm::Dictionary>(config_->get_libraries_root(), 256);
+    compute_phase_cfg->libraries = std::make_unique<vm::Dictionary>(config->get_libraries_root(), 256);
-    compute_phase_cfg_.global_config = config_->get_root_cell();
+    compute_phase_cfg->global_config = config->get_root_cell();
   }
   {
     // compute action_phase_cfg
     block::gen::MsgForwardPrices::Record rec;
-    auto cell = config_->get_config_param(24);
+    auto cell = config->get_config_param(24);
     if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) {
-      return fatal_error("cannot fetch masterchain message transfer prices from masterchain configuration");
+      return td::Status::Error(-668, "cannot fetch masterchain message transfer prices from masterchain configuration");
     }
-    action_phase_cfg_.fwd_mc =
+    action_phase_cfg->fwd_mc =
         block::MsgPrices{rec.lump_price,           rec.bit_price,          rec.cell_price, rec.ihr_price_factor,
                          (unsigned)rec.first_frac, (unsigned)rec.next_frac};
-    cell = config_->get_config_param(25);
+    cell = config->get_config_param(25);
     if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) {
-      return fatal_error("cannot fetch standard message transfer prices from masterchain configuration");
+      return td::Status::Error(-668, "cannot fetch standard message transfer prices from masterchain configuration");
     }
-    action_phase_cfg_.fwd_std =
+    action_phase_cfg->fwd_std =
         block::MsgPrices{rec.lump_price,           rec.bit_price,          rec.cell_price, rec.ihr_price_factor,
                          (unsigned)rec.first_frac, (unsigned)rec.next_frac};
-    action_phase_cfg_.workchains = &config_->get_workchain_list();
+    action_phase_cfg->workchains = &config->get_workchain_list();
-    action_phase_cfg_.bounce_msg_body = (config_->has_capability(ton::capBounceMsgBody) ? 256 : 0);
+    action_phase_cfg->bounce_msg_body = (config->has_capability(ton::capBounceMsgBody) ? 256 : 0);
   }
   {
     // fetch block_grams_created
-    auto cell = config_->get_config_param(14);
+    auto cell = config->get_config_param(14);
     if (cell.is_null()) {
-      basechain_create_fee_ = masterchain_create_fee_ = td::zero_refint();
+      *basechain_create_fee = *masterchain_create_fee = td::zero_refint();
     } else {
       block::gen::BlockCreateFees::Record create_fees;
       if (!(tlb::unpack_cell(cell, create_fees) &&
-            block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, masterchain_create_fee_) &&
+            block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, *masterchain_create_fee) &&
-            block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, basechain_create_fee_))) {
+            block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, *basechain_create_fee))) {
-        return fatal_error("cannot unpack BlockCreateFees from configuration parameter #14");
+        return td::Status::Error(-668, "cannot unpack BlockCreateFees from configuration parameter #14");
       }
     }
   }
-  return true;
+  return std::move(config);
 }
 
 bool Collator::compute_minted_amount(block::CurrencyCollection& to_mint) {
@@ -1920,7 +1944,6 @@ std::unique_ptr<block::Account> Collator::make_account_from(td::ConstBitPtr addr
   }
   auto ptr = std::make_unique<block::Account>(workchain(), addr);
   if (account.is_null()) {
-    ptr->created = true;
     if (!ptr->init_new(now_)) {
       return nullptr;
     }
@@ -2218,75 +2241,25 @@ Ref<vm::Cell> Collator::create_ordinary_transaction(Ref<vm::Cell> msg_root) {
   }
   block::Account* acc = acc_res.move_as_ok();
   assert(acc);
-  if (acc->last_trans_end_lt_ >= start_lt && acc->transactions.empty()) {
+
-    fatal_error(PSTRING() << "last transaction time in the state of account " << workchain() << ":" << addr.to_hex()
+
-                          << " is too large");
+  auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt,
+                                                    &storage_phase_cfg_, &compute_phase_cfg_,
+                                                    &action_phase_cfg_,
+                                                    external, last_proc_int_msg_.first
+                                                   );
+  if(res.is_error()) {
+    auto error = res.move_as_error();
+    if(error.code() == -701) {
+      // ignorable errors
+      LOG(DEBUG) << error.message();
       return {};
     }
-  auto trans_min_lt = start_lt;
+    fatal_error(std::move(error));
-  if (external) {
-    // transactions processing external messages must have lt larger than all processed internal messages
-    trans_min_lt = std::max(trans_min_lt, last_proc_int_msg_.first);
-  }
-  std::unique_ptr<block::Transaction> trans =
-      std::make_unique<block::Transaction>(*acc, block::Transaction::tr_ord, trans_min_lt + 1, now_, msg_root);
-  bool ihr_delivered = false;  // FIXME
-  if (!trans->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) {
-    if (external) {
-      // inbound external message was not accepted
-      LOG(DEBUG) << "inbound external message rejected by account " << addr.to_hex()
-                 << " before smart-contract execution";
-      return {};
-    }
-    fatal_error("cannot unpack input message for a new transaction");
-    return {};
-  }
-  if (trans->bounce_enabled) {
-    if (!trans->prepare_storage_phase(storage_phase_cfg_, true)) {
-      fatal_error("cannot create storage phase of a new transaction for smart contract "s + addr.to_hex());
-      return {};
-    }
-    if (!external && !trans->prepare_credit_phase()) {
-      fatal_error("cannot create credit phase of a new transaction for smart contract "s + addr.to_hex());
-      return {};
-    }
-  } else {
-    if (!external && !trans->prepare_credit_phase()) {
-      fatal_error("cannot create credit phase of a new transaction for smart contract "s + addr.to_hex());
-      return {};
-    }
-    if (!trans->prepare_storage_phase(storage_phase_cfg_, true, true)) {
-      fatal_error("cannot create storage phase of a new transaction for smart contract "s + addr.to_hex());
-      return {};
-    }
-  }
-  if (!trans->prepare_compute_phase(compute_phase_cfg_)) {
-    fatal_error("cannot create compute phase of a new transaction for smart contract "s + addr.to_hex());
-    return {};
-  }
-  if (!trans->compute_phase->accepted) {
-    if (external) {
-      // inbound external message was not accepted
-      LOG(DEBUG) << "inbound external message rejected by transaction " << addr.to_hex();
-      return {};
-    } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) {
-      fatal_error("new ordinary transaction for smart contract "s + addr.to_hex() +
-                  " has not been accepted by the smart contract (?)");
-      return {};
-    }
-  }
-  if (trans->compute_phase->success && !trans->prepare_action_phase(action_phase_cfg_)) {
-    fatal_error("cannot create action phase of a new transaction for smart contract "s + addr.to_hex());
-    return {};
-  }
-  if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(action_phase_cfg_)) {
-    fatal_error("cannot create bounce phase of a new transaction for smart contract "s + addr.to_hex());
-    return {};
-  }
-  if (!trans->serialize()) {
-    fatal_error("cannot serialize new transaction for smart contract "s + addr.to_hex());
     return {};
   }
+  std::unique_ptr<block::Transaction> trans = res.move_as_ok();
+
   if (!trans->update_limits(*block_limit_status_)) {
     fatal_error("cannot update block limit status to include the new transaction");
     return {};
@@ -2296,15 +2269,81 @@ Ref<vm::Cell> Collator::create_ordinary_transaction(Ref<vm::Cell> msg_root) {
     fatal_error("cannot commit new transaction for smart contract "s + addr.to_hex());
     return {};
   }
+
   register_new_msgs(*trans);
   update_max_lt(acc->last_trans_end_lt_);
-  // temporary patch to stop producing dangerous block
-  if (acc->status == block::Account::acc_nonexist) {
-    block_full_ = true;
-  }
   return trans_root;
 }
 
+// If td::status::error_code == 669 - Fatal Error block can not be produced
+// if td::status::error_code == 701 - Transaction can not be included into block, but it's ok (external or too early internal)
+td::Result<std::unique_ptr<block::Transaction>> Collator::impl_create_ordinary_transaction(Ref<vm::Cell> msg_root,
+                                                         block::Account* acc,
+                                                         UnixTime utime, LogicalTime lt,
+                                                         block::StoragePhaseConfig* storage_phase_cfg,
+                                                         block::ComputePhaseConfig* compute_phase_cfg,
+                                                         block::ActionPhaseConfig* action_phase_cfg,
+                                                         bool external, LogicalTime after_lt) {
+  if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) {
+    return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex()
+                          << " is too large");
+  }
+  auto trans_min_lt = lt;
+  if (external) {
+    // transactions processing external messages must have lt larger than all processed internal messages
+    trans_min_lt = std::max(trans_min_lt, after_lt);
+  }
+
+  std::unique_ptr<block::Transaction> trans =
+      std::make_unique<block::Transaction>(*acc, block::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root);
+  bool ihr_delivered = false;  // FIXME
+  if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) {
+    if (external) {
+      // inbound external message was not accepted
+      return td::Status::Error(-701,"inbound external message rejected by account "s + acc->addr.to_hex() +
+                                                           " before smart-contract execution");
+      }
+    return td::Status::Error(-669,"cannot unpack input message for a new transaction");
+  }
+  if (trans->bounce_enabled) {
+    if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) {
+      return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex());
+      }
+    if (!external && !trans->prepare_credit_phase()) {
+      return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex());
+      }
+  } else {
+    if (!external && !trans->prepare_credit_phase()) {
+      return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex());
+      }
+    if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) {
+      return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex());
+      }
+  }
+  if (!trans->prepare_compute_phase(*compute_phase_cfg)) {
+    return td::Status::Error(-669,"cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex());
+  }
+  if (!trans->compute_phase->accepted) {
+    if (external) {
+      // inbound external message was not accepted
+        return td::Status::Error(-701,"inbound external message rejected by transaction "s + acc->addr.to_hex());
+      } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) {
+        return td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() +
+                  " has not been accepted by the smart contract (?)");
+      }
+  }
+  if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) {
+    return td::Status::Error(-669,"cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex());
+  }
+  if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(*action_phase_cfg)) {
+    return td::Status::Error(-669,"cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex());
+  }
+  if (!trans->serialize()) {
+    return td::Status::Error(-669,"cannot serialize new transaction for smart contract "s + acc->addr.to_hex());
+  }
+  return std::move(trans);
+}
+
 void Collator::update_max_lt(ton::LogicalTime lt) {
   CHECK(lt >= start_lt);
   if (lt > max_lt) {
@@ -2451,10 +2490,6 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R
   if (!insert_out_msg(cb.finalize())) {
     return -1;
   }
-  // 6.5. check for temporary patch can be left here
-  if (block_full_) {
-    return 3;
-  }
   // 7. check whether the block is full now
   if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) {
     block_full_ = true;

validator/impl/external-message.cpp

@@ -16,19 +16,25 @@
 
     Copyright 2017-2020 Telegram Systems LLP
 */
+
 #include "external-message.hpp"
+#include "collator-impl.h"
 #include "vm/boc.h"
 #include "block/block-parse.h"
 #include "block/block-auto.h"
 #include "block/block-db.h"
+#include "fabric.h"
+#include "td/actor/actor.h"
+#include "td/utils/Random.h"
+#include "crypto/openssl/rand.hpp"
 
 namespace ton {
 
 namespace validator {
 using td::Ref;
 
-ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref<vm::Cell> root, AccountIdPrefixFull addr_prefix)
+ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref<vm::Cell> root, AccountIdPrefixFull addr_prefix, ton::WorkchainId wc, ton::StdSmcAddress addr)
-    : root_(std::move(root)), addr_prefix_(addr_prefix), data_(std::move(data)) {
+    : root_(std::move(root)), addr_prefix_(addr_prefix), data_(std::move(data)), wc_(wc), addr_(addr) {
   hash_ = block::compute_file_hash(data_);
 }
 
@@ -70,7 +76,95 @@ td::Result<Ref<ExtMessageQ>> ExtMessageQ::create_ext_message(td::BufferSlice dat
   if (!dest_prefix.is_valid()) {
     return td::Status::Error("destination of an inbound external message is an invalid blockchain address");
   }
-  return Ref<ExtMessageQ>{true, std::move(data), std::move(ext_msg), dest_prefix};
+  ton::StdSmcAddress addr;
+  ton::WorkchainId wc;
+  if(!block::tlb::t_MsgAddressInt.extract_std_address(info.dest, wc, addr)) {
+    return td::Status::Error(PSLICE() << "Can't parse destination address");
+  }
+
+  return Ref<ExtMessageQ>{true, std::move(data), std::move(ext_msg), dest_prefix, wc, addr};
+}
+
+void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId<ton::validator::ValidatorManager> manager,
+                              td::Promise<td::Unit> promise) {
+  auto R = create_ext_message(std::move(data));
+  if (R.is_error()) {
+    return promise.set_error(R.move_as_error_prefix("failed to parse external message "));
+  }
+  auto M = R.move_as_ok();
+  auto root = M->root_cell();
+  block::gen::CommonMsgInfo::Record_ext_in_msg_info info;
+  tlb::unpack_cell_inexact(root, info); // checked in create message
+  ton::StdSmcAddress addr = M->addr();
+  ton::WorkchainId wc = M->wc();
+
+  run_fetch_account_state(wc, addr, manager,
+      [promise = std::move(promise), msg_root = root, wc = wc](td::Result<std::tuple<td::Ref<vm::CellSlice>,UnixTime,LogicalTime,std::unique_ptr<block::ConfigInfo>>> res) mutable {
+        if (res.is_error()) {
+          promise.set_error(td::Status::Error(PSLICE() << "Failed to get account state"));
+        } else {
+          auto tuple = res.move_as_ok();
+          block::Account acc;
+          auto shard_acc = std::move(std::get<0>(tuple));
+          auto utime = std::get<1>(tuple);
+          auto lt = std::get<2>(tuple);
+          auto config = std::move(std::get<3>(tuple));
+          if(!acc.unpack(shard_acc, {}, utime, false)) {
+            promise.set_error(td::Status::Error(PSLICE() << "Failed to unpack account state"));
+          } else {
+            if(run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config))) {
+              promise.set_value(td::Unit());
+            } else {
+              promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted"));
+            }
+          }
+        }
+      }
+  );
+}
+
+bool ExtMessageQ::run_message_on_account(ton::WorkchainId wc,
+                                         block::Account* acc,
+                                         UnixTime utime, LogicalTime lt,
+                                         td::Ref<vm::Cell> msg_root,
+                                         std::unique_ptr<block::ConfigInfo> config) {
+
+   Ref<vm::Cell> old_mparams;
+   std::vector<block::StoragePrices> storage_prices_;
+   block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_};
+   td::BitArray<256> rand_seed_;
+   block::ComputePhaseConfig compute_phase_cfg_;
+   block::ActionPhaseConfig action_phase_cfg_;
+   td::RefInt256 masterchain_create_fee, basechain_create_fee;
+
+   auto fetch_res = Collator::impl_fetch_config_params(std::move(config), &old_mparams,
+                                                 &storage_prices_, &storage_phase_cfg_,
+                                                 &rand_seed_, &compute_phase_cfg_,
+                                                 &action_phase_cfg_, &masterchain_create_fee,
+                                                 &basechain_create_fee, wc);
+   if(fetch_res.is_error()) {
+    auto error = fetch_res.move_as_error();
+    LOG(DEBUG) << "Cannot fetch config params" << error.message();
+    return false;
+   }
+
+   auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt,
+                                                    &storage_phase_cfg_, &compute_phase_cfg_,
+                                                    &action_phase_cfg_,
+                                                    true, lt);
+   if(res.is_error()) {
+    auto error = res.move_as_error();
+    LOG(DEBUG) << "Cannot run message on account" << error.message();
+    return false;
+   }
+   std::unique_ptr<block::Transaction> trans = res.move_as_ok();
+
+   auto trans_root = trans->commit(*acc);
+   if (trans_root.is_null()) {
+     LOG(DEBUG) << "cannot commit new transaction for smart contract ";
+     return false;
+   }
+   return true;
 }
 
 }  // namespace validator

validator/impl/external-message.hpp

@@ -18,9 +18,11 @@
 */
 #pragma once
 
+#include "interfaces/validator-manager.h"
 #include "validator/interfaces/external-message.h"
 #include "auto/tl/ton_api.h"
 #include "adnl/utils.hpp"
+#include "block/transaction.h"
 
 namespace ton {
 
@@ -31,6 +33,8 @@ class ExtMessageQ : public ExtMessage {
   AccountIdPrefixFull addr_prefix_;
   td::BufferSlice data_;
   Hash hash_;
+  ton::WorkchainId wc_;
+  ton::StdSmcAddress addr_;
 
  public:
   static constexpr unsigned max_ext_msg_size = 65535;
@@ -47,8 +51,23 @@ class ExtMessageQ : public ExtMessage {
   Hash hash() const override {
     return hash_;
   }
-  ExtMessageQ(td::BufferSlice data, td::Ref<vm::Cell> root, AccountIdPrefixFull shard);
+  ton::WorkchainId wc() const override {
+    return wc_;
+  }
+
+  ton::StdSmcAddress addr() const override {
+    return addr_;
+  }
+
+  ExtMessageQ(td::BufferSlice data, td::Ref<vm::Cell> root, AccountIdPrefixFull shard, ton::WorkchainId wc, ton::StdSmcAddress addr);
   static td::Result<td::Ref<ExtMessageQ>> create_ext_message(td::BufferSlice data);
+  static void run_message(td::BufferSlice data, td::actor::ActorId<ton::validator::ValidatorManager> manager,
+                        td::Promise<td::Unit> promise);
+  static bool run_message_on_account(ton::WorkchainId wc,
+                                         block::Account* acc,
+                                         UnixTime utime, LogicalTime lt,
+                                         td::Ref<vm::Cell> msg_root,
+                                         std::unique_ptr<block::ConfigInfo> config);
 };
 
 }  // namespace validator

validator/impl/fabric.cpp

@@ -116,6 +116,10 @@ td::Result<td::Ref<ExtMessage>> create_ext_message(td::BufferSlice data) {
   return std::move(res);
 }
 
+void run_check_external_message(td::BufferSlice data, td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise) {
+  ExtMessageQ::run_message(std::move(data), std::move(manager), std::move(promise));
+}
+
 td::Result<td::Ref<IhrMessage>> create_ihr_message(td::BufferSlice data) {
   TRY_RESULT(res, IhrMessageQ::create_ihr_message(std::move(data)));
   return std::move(res);
@@ -237,6 +241,11 @@ void run_liteserver_query(td::BufferSlice data, td::actor::ActorId<ValidatorMana
   LiteQuery::run_query(std::move(data), std::move(manager), std::move(promise));
 }
 
+void run_fetch_account_state(WorkchainId wc, StdSmcAddress  addr, td::actor::ActorId<ValidatorManager> manager,
+                             td::Promise<std::tuple<td::Ref<vm::CellSlice>,UnixTime,LogicalTime,std::unique_ptr<block::ConfigInfo>>> promise) {
+  LiteQuery::fetch_account_state(wc, addr, std::move(manager), std::move(promise));
+}
+
 void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block,
                                           td::Ref<MasterchainState> masterchain_state,
                                           td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,

validator/impl/liteserver.cpp

@@ -58,15 +58,28 @@ void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId<ValidatorMana
   td::actor::create_actor<LiteQuery>("litequery", std::move(data), std::move(manager), std::move(promise)).release();
 }
 
+void LiteQuery::fetch_account_state(WorkchainId wc, StdSmcAddress  acc_addr, td::actor::ActorId<ton::validator::ValidatorManager> manager,
+                                 td::Promise<std::tuple<td::Ref<vm::CellSlice>,UnixTime,LogicalTime,std::unique_ptr<block::ConfigInfo>>> promise) {
+  td::actor::create_actor<LiteQuery>("litequery", wc, acc_addr, std::move(manager), std::move(promise)).release();
+}
+
 LiteQuery::LiteQuery(td::BufferSlice data, td::actor::ActorId<ValidatorManager> manager,
                      td::Promise<td::BufferSlice> promise)
     : query_(std::move(data)), manager_(std::move(manager)), promise_(std::move(promise)) {
   timeout_ = td::Timestamp::in(default_timeout_msec * 0.001);
 }
 
+LiteQuery::LiteQuery(WorkchainId wc, StdSmcAddress  acc_addr, td::actor::ActorId<ValidatorManager> manager,
+                     td::Promise<std::tuple<td::Ref<vm::CellSlice>,UnixTime,LogicalTime,std::unique_ptr<block::ConfigInfo>>> promise)
+    : manager_(std::move(manager)), acc_state_promise_(std::move(promise)), acc_workchain_(wc), acc_addr_(acc_addr) {
+  timeout_ = td::Timestamp::in(default_timeout_msec * 0.001);
+}
+
 void LiteQuery::abort_query(td::Status reason) {
   LOG(INFO) << "aborted liteserver query: " << reason.to_string();
-  if (promise_) {
+  if (acc_state_promise_) {
+    acc_state_promise_.set_error(std::move(reason));
+  } else if (promise_) {
     promise_.set_error(std::move(reason));
   }
   stop();
@@ -111,6 +124,11 @@ bool LiteQuery::finish_query(td::BufferSlice result) {
 void LiteQuery::start_up() {
   alarm_timestamp() = timeout_;
 
+  if(acc_state_promise_) {
+    td::actor::send_closure_later(actor_id(this),&LiteQuery::perform_fetchAccountState);
+    return;
+  }
+
   auto F = fetch_tl_object<ton::lite_api::Function>(std::move(query_), true);
   if (F.is_error()) {
     abort_query(F.move_as_error());
@@ -180,6 +198,9 @@ void LiteQuery::start_up() {
             this->perform_runSmcMethod(ton::create_block_id(q.id_), static_cast<WorkchainId>(q.account_->workchain_),
                                        q.account_->id_, q.mode_, q.method_id_, std::move(q.params_));
           },
+          [&](lite_api::liteServer_getLibraries& q) {
+            this->perform_getLibraries(q.library_list_);
+          },
           [&](auto& obj) { this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); }));
 }
 
@@ -205,17 +226,23 @@ void LiteQuery::perform_getMasterchainInfo(int mode) {
   }
   td::actor::send_closure_later(
       manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block,
-      [Self = actor_id(this), mode](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res) {
+      [Self = actor_id(this), return_state = bool(acc_state_promise_), mode](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res) {
         if (res.is_error()) {
           td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
         } else {
           auto pair = res.move_as_ok();
-          td::actor::send_closure_later(Self, &LiteQuery::continue_getMasterchainInfo, std::move(pair.first),
+          auto func = return_state ? &LiteQuery::gotMasterchainInfoForAccountState : &LiteQuery::continue_getMasterchainInfo;
+          td::actor::send_closure_later(Self, func, std::move(pair.first),
                                         pair.second, mode);
         }
       });
 }
 
+void LiteQuery::gotMasterchainInfoForAccountState(Ref<ton::validator::MasterchainState> mc_state, BlockIdExt blkid,
+                                            int mode) {
+  perform_getAccountState(blkid, acc_workchain_, acc_addr_, 0x80000000);
+}
+
 void LiteQuery::continue_getMasterchainInfo(Ref<ton::validator::MasterchainState> mc_state, BlockIdExt blkid,
                                             int mode) {
   LOG(INFO) << "obtained data for getMasterchainInfo() : last block = " << blkid.to_str();
@@ -431,15 +458,25 @@ void LiteQuery::continue_getZeroState(BlockIdExt blkid, td::BufferSlice state) {
 
 void LiteQuery::perform_sendMessage(td::BufferSlice data) {
   LOG(INFO) << "started a sendMessage(<" << data.size() << " bytes>) liteserver query";
-  auto res = ton::validator::create_ext_message(std::move(data));
+ td::actor::send_closure_later(
-  if (res.is_error()) {
+      manager_, &ValidatorManager::check_external_message, data.clone(),
-    abort_query(res.move_as_error());
+      [Self = actor_id(this), data = std::move(data), manager = manager_](td::Result<td::Unit> res) {
-    return;
+        if(res.is_error()) {
+           td::actor::send_closure(Self, &LiteQuery::abort_query,
+                                    res.move_as_error_prefix("cannot apply external message to current state : "s));
+        } else {
+          auto crm = ton::validator::create_ext_message(data.clone());
+          if (crm.is_error()) {
+             //UNREACHABLE, checks in check_external_message,
+             td::actor::send_closure(Self, &LiteQuery::abort_query,
+                        crm.move_as_error());
           }
           LOG(INFO) << "sending an external message to validator manager";
-  td::actor::send_closure_later(manager_, &ValidatorManager::send_external_message, res.move_as_ok());
+          td::actor::send_closure_later(manager, &ValidatorManager::send_external_message, crm.move_as_ok());
           auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_sendMsgStatus>(1);
-  finish_query(std::move(b));
+          td::actor::send_closure(Self, &LiteQuery::finish_query, std::move(b));
+        }
+      });
 }
 
 bool LiteQuery::request_mc_block_data(BlockIdExt blkid) {
@@ -702,6 +739,10 @@ void LiteQuery::continue_getAccountState_0(Ref<ton::validator::MasterchainState>
   request_mc_block_data(blkid);
 }
 
+void LiteQuery::perform_fetchAccountState() {
+  perform_getMasterchainInfo(-1);
+}
+
 void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode,
                                      td::int64 method_id, td::BufferSlice params) {
   LOG(INFO) << "started a runSmcMethod(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ", "
@@ -743,6 +784,73 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St
   perform_getAccountState(blkid, workchain, addr, mode | 0x10000);
 }
 
+void LiteQuery::perform_getLibraries(std::vector<td::Bits256> library_list) {
+  LOG(INFO) << "started a getLibraries(<list of " << library_list.size() << " parameters>) liteserver query";
+  if (library_list.size() > 16) {
+    LOG(INFO) << "too many libraries requested, returning only first 16";
+    library_list.resize(16);
+  }
+  sort( library_list.begin(), library_list.end() );
+  library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() );
+  td::actor::send_closure_later(
+      manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block,
+      [Self = actor_id(this), library_list](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res) -> void {
+        if (res.is_error()) {
+          td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
+        } else {
+          auto pair = res.move_as_ok();
+          td::actor::send_closure_later(Self, &LiteQuery::continue_getLibraries, std::move(pair.first),
+                                        pair.second, library_list);
+        }
+      });
+}
+
+void LiteQuery::continue_getLibraries(Ref<ton::validator::MasterchainState> mc_state, BlockIdExt blkid, std::vector<td::Bits256> library_list) {
+  LOG(INFO) << "obtained last masterchain block = " << blkid.to_str();
+  base_blk_id_ = blkid;
+  CHECK(mc_state.not_null());
+  mc_state_ = Ref<MasterchainStateQ>(std::move(mc_state));
+  CHECK(mc_state_.not_null());
+
+  auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries);
+  if (rconfig.is_error()) {
+    fatal_error("cannot extract library list block configuration from masterchain state");
+    return;
+  }
+  auto config = rconfig.move_as_ok();
+
+  if (false) {
+    std::ostringstream os;
+    vm::load_cell_slice(config->get_libraries_root()).print_rec(os);
+    LOG(INFO) << "\n" << os.str();
+
+    auto lib_dict = std::make_unique<vm::Dictionary>(config->get_libraries_root(), 256);
+    for (auto k: *lib_dict) {
+      std::ostringstream oss;
+      k.second->print_rec(oss);
+      LOG(INFO) << "library " << k.first.to_hex(256) << ": \n" << oss.str();
+    }
+  }
+
+  std::vector<ton::tl_object_ptr<ton::lite_api::liteServer_libraryEntry>> a;
+  for (const auto& hash : library_list) {
+    LOG(INFO) << "looking for library " << hash.to_hex();
+    auto libres = config->lookup_library(hash);
+    if (libres.is_null()) {
+      LOG(INFO) << "library lookup result is null";
+      continue;
+    }
+    auto data = vm::std_boc_serialize(libres);
+    if (data.is_error()) {
+      LOG(WARNING) << "library serialization failed: " << data.move_as_error().to_string();
+      continue;
+    }
+    a.push_back(ton::create_tl_object<ton::lite_api::liteServer_libraryEntry>(hash, data.move_as_ok()));
+  }
+  auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_libraryResult>(std::move(a));
+  finish_query(std::move(b));
+}
+
 void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt) {
   LOG(INFO) << "started a getOneTransaction(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ","
             << lt << ") liteserver query";
@@ -1010,6 +1118,19 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) {
   }
   vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, block::tlb::aug_ShardAccounts};
   auto acc_csr = accounts_dict.lookup(acc_addr_);
+  if (mode_ & 0x80000000) {
+    auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), 0xFFFF);
+    if (config.is_error()) {
+      fatal_error(config.move_as_error());
+      return;
+    }
+    auto rconfig = config.move_as_ok();
+    acc_state_promise_.set_value(std::make_tuple(
+                                  std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig)
+                                 ));
+    return;
+  }
+
   Ref<vm::Cell> acc_root;
   if (acc_csr.not_null()) {
     acc_root = acc_csr->prefetch_ref();
@@ -1099,7 +1220,7 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice
   long long gas_limit = client_method_gas_limit;
   LOG(DEBUG) << "creating VM with gas limit " << gas_limit;
   // **** INIT VM ****
-  vm::GasLimits gas{gas_limit};
+  vm::GasLimits gas{gas_limit, gas_limit};
   vm::VmState vm{std::move(code), std::move(stack_), gas, 1, std::move(data), vm::VmLog::Null()};
   auto c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref<vm::CellSlice>(acc.addr->clone()), balance);
   vm.set_c7(c7);  // tuple with SmartContractInfo

validator/impl/liteserver.hpp

@@ -26,6 +26,8 @@
 #include "block.hpp"
 #include "shard.hpp"
 #include "proof.hpp"
+#include "block/block-auto.h"
+
 
 namespace ton {
 
@@ -37,6 +39,9 @@ class LiteQuery : public td::actor::Actor {
   td::actor::ActorId<ton::validator::ValidatorManager> manager_;
   td::Timestamp timeout_;
   td::Promise<td::BufferSlice> promise_;
+
+  td::Promise<std::tuple<td::Ref<vm::CellSlice>,UnixTime,LogicalTime,std::unique_ptr<block::ConfigInfo>>> acc_state_promise_;
+
   int pending_{0};
   int mode_{0};
   WorkchainId acc_workchain_;
@@ -71,9 +76,14 @@ class LiteQuery : public td::actor::Actor {
   };  // version 1.1; +1 = build block proof chains, +2 = masterchainInfoExt, +4 = runSmcMethod
   LiteQuery(td::BufferSlice data, td::actor::ActorId<ton::validator::ValidatorManager> manager,
             td::Promise<td::BufferSlice> promise);
+  LiteQuery(WorkchainId wc, StdSmcAddress  acc_addr, td::actor::ActorId<ton::validator::ValidatorManager> manager,
+            td::Promise<std::tuple<td::Ref<vm::CellSlice>,UnixTime,LogicalTime,std::unique_ptr<block::ConfigInfo>>> promise);
   static void run_query(td::BufferSlice data, td::actor::ActorId<ton::validator::ValidatorManager> manager,
                         td::Promise<td::BufferSlice> promise);
 
+  static void fetch_account_state(WorkchainId wc, StdSmcAddress  acc_addr, td::actor::ActorId<ton::validator::ValidatorManager> manager,
+                                  td::Promise<std::tuple<td::Ref<vm::CellSlice>,UnixTime,LogicalTime,std::unique_ptr<block::ConfigInfo>>> promise);
+
  private:
   bool fatal_error(td::Status error);
   bool fatal_error(std::string err_msg, int err_code = -400);
@@ -87,6 +97,7 @@ class LiteQuery : public td::actor::Actor {
   void perform_getVersion();
   void perform_getMasterchainInfo(int mode);
   void continue_getMasterchainInfo(Ref<MasterchainState> mc_state, BlockIdExt blkid, int mode);
+  void gotMasterchainInfoForAccountState(Ref<MasterchainState> mc_state, BlockIdExt blkid, int mode);
   void perform_getBlock(BlockIdExt blkid);
   void continue_getBlock(BlockIdExt blkid, Ref<BlockData> block);
   void perform_getBlockHeader(BlockIdExt blkid, int mode);
@@ -99,10 +110,13 @@ class LiteQuery : public td::actor::Actor {
   void continue_getAccountState_0(Ref<MasterchainState> mc_state, BlockIdExt blkid);
   void continue_getAccountState();
   void finish_getAccountState(td::BufferSlice shard_proof);
+  void perform_fetchAccountState();
   void perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode, td::int64 method_id,
                             td::BufferSlice params);
   void finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice state_proof, Ref<vm::Cell> acc_root,
                            UnixTime gen_utime, LogicalTime gen_lt);
+  void perform_getLibraries(std::vector<td::Bits256> library_list);
+  void continue_getLibraries(Ref<MasterchainState> mc_state, BlockIdExt blkid, std::vector<td::Bits256> library_list);
   void perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt);
   void continue_getOneTransaction();
   void perform_getTransactions(WorkchainId workchain, StdSmcAddress addr, LogicalTime lt, Bits256 hash, unsigned count);

validator/impl/shard.cpp

@@ -26,6 +26,7 @@
 #include "vm/cells/MerkleUpdate.h"
 #include "block/block-parse.h"
 #include "block/block-auto.h"
+#include "td/utils/filesystem.h"
 
 #define LAZY_STATE_DESERIALIZE 1
 
@@ -301,6 +302,30 @@ td::Result<td::BufferSlice> ShardStateQ::serialize() const {
   return st_res.move_as_ok();
 }
 
+td::Status ShardStateQ::serialize_to_file(td::FileFd& fd) const {
+  td::PerfWarningTimer perf_timer_{"serializestate", 0.1};
+  if (!data.is_null()) {
+    auto cur_data = data.clone();
+    while (cur_data.size() > 0) {
+      TRY_RESULT(s, fd.write(cur_data.as_slice()));
+      cur_data.confirm_read(s);
+    }
+    return td::Status::OK();
+  }
+  if (root.is_null()) {
+    return td::Status::Error(-666, "cannot serialize an uninitialized state");
+  }
+  vm::BagOfCells new_boc;
+  new_boc.set_root(root);
+  TRY_STATUS(new_boc.import_cells());
+  auto st_res = new_boc.serialize_to_file(fd, 31);
+  if (st_res.is_error()) {
+    LOG(ERROR) << "cannot serialize a shardchain state";
+    return st_res.move_as_error();
+  }
+  return td::Status::OK();
+}
+
 MasterchainStateQ::MasterchainStateQ(const BlockIdExt& _id, td::BufferSlice _data)
     : MasterchainState(), ShardStateQ(_id, std::move(_data)) {
 }

validator/impl/shard.hpp

@@ -87,6 +87,7 @@ class ShardStateQ : virtual public ShardState {
   td::Result<Ref<ShardState>> merge_with(const ShardState& with) const override;
   td::Result<std::pair<Ref<ShardState>, Ref<ShardState>>> split() const override;
   td::Result<td::BufferSlice> serialize() const override;
+  td::Status serialize_to_file(td::FileFd& fd) const override;
 };
 
 #if TD_MSVC

validator/impl/validate-query.cpp

@@ -4073,7 +4073,6 @@ std::unique_ptr<block::Account> ValidateQuery::make_account_from(td::ConstBitPtr
                                                                  Ref<vm::CellSlice> extra) {
   auto ptr = std::make_unique<block::Account>(workchain(), addr);
   if (account.is_null()) {
-    ptr->created = true;
     if (!ptr->init_new(now_)) {
       return nullptr;
     }
@@ -4308,7 +4307,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT
     }
   }
   if (is_first && is_masterchain() && account.is_special && account.tick &&
-      (tag != block::gen::TransactionDescr::trans_tick_tock || (td_cs.prefetch_ulong(4) & 1)) && !account.created) {
+      (tag != block::gen::TransactionDescr::trans_tick_tock || (td_cs.prefetch_ulong(4) & 1)) && account.orig_status == block::Account::acc_active) {
     return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
                                   << " is the first transaction for this special tick account in this block, but the "
                                      "transaction is not a tick transaction");

validator/interfaces/db.h

@@ -53,6 +53,9 @@ class Db : public td::actor::Actor {
 
   virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
                                            td::Promise<td::Unit> promise) = 0;
+  virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                               std::function<td::Status(td::FileFd&)> write_data,
+                                               td::Promise<td::Unit> promise) = 0;
   virtual void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id,
                                          td::Promise<td::BufferSlice> promise) = 0;
   virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,

validator/interfaces/external-message.h

@@ -35,6 +35,8 @@ class ExtMessage : public td::CntObject {
   virtual td::BufferSlice serialize() const = 0;
   virtual td::Ref<vm::Cell> root_cell() const = 0;
   virtual Hash hash() const = 0;
+  virtual ton::WorkchainId wc() const = 0;
+  virtual ton::StdSmcAddress addr() const = 0;
 };
 
 }  // namespace validator

validator/interfaces/shard.h

@@ -55,6 +55,7 @@ class ShardState : public td::CntObject {
   virtual td::Result<std::pair<td::Ref<ShardState>, td::Ref<ShardState>>> split() const = 0;
 
   virtual td::Result<td::BufferSlice> serialize() const = 0;
+  virtual td::Status serialize_to_file(td::FileFd& fd) const = 0;
 };
 
 class MasterchainState : virtual public ShardState {

validator/interfaces/validator-manager.h

@@ -57,6 +57,9 @@ class ValidatorManager : public ValidatorManagerInterface {
                                td::Promise<td::Ref<ShardState>> promise) = 0;
   virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
                                            td::Promise<td::Unit> promise) = 0;
+  virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                               std::function<td::Status(td::FileFd&)> write_data,
+                                               td::Promise<td::Unit> promise) = 0;
   virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise<td::Unit> promise) = 0;
   virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout,
                                 td::Promise<td::Ref<ShardState>> promise) = 0;

validator/manager-disk.cpp

@@ -680,6 +680,13 @@ void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, Bloc
                           std::move(promise));
 }
 
+void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                                           std::function<td::Status(td::FileFd&)> write_data,
+                                                           td::Promise<td::Unit> promise) {
+  td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id,
+                          std::move(write_data), std::move(promise));
+}
+
 void ValidatorManagerImpl::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state,
                                                  td::Promise<td::Unit> promise) {
   td::actor::send_closure(db_, &Db::store_zero_state_file, block_id, std::move(state), std::move(promise));

validator/manager-disk.hpp

@@ -124,6 +124,9 @@ class ValidatorManagerImpl : public ValidatorManager {
   //void get_block_description(BlockIdExt block_id, td::Promise<BlockDescription> promise) override;
 
   void new_external_message(td::BufferSlice data) override;
+  void check_external_message(td::BufferSlice data, td::Promise<td::Unit> promise) override {
+    UNREACHABLE();
+  }
   void new_ihr_message(td::BufferSlice data) override;
   void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override;
 
@@ -140,6 +143,9 @@ class ValidatorManagerImpl : public ValidatorManager {
                        td::Promise<td::Ref<ShardState>> promise) override;
   void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
                                    td::Promise<td::Unit> promise) override;
+  void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                       std::function<td::Status(td::FileFd&)> write_data,
+                                       td::Promise<td::Unit> promise) override;
   void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise<td::Unit> promise) override;
   void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout,
                         td::Promise<td::Ref<ShardState>> promise) override;

validator/manager-hardfork.hpp

@@ -144,6 +144,9 @@ class ValidatorManagerImpl : public ValidatorManager {
   void get_key_block_proof_link(BlockIdExt block_id, td::Promise<td::BufferSlice> promise) override;
 
   void new_external_message(td::BufferSlice data) override;
+  void check_external_message(td::BufferSlice data, td::Promise<td::Unit> promise) override {
+    UNREACHABLE();
+  }
   void new_ihr_message(td::BufferSlice data) override;
   void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override {
     UNREACHABLE();
@@ -166,6 +169,11 @@ class ValidatorManagerImpl : public ValidatorManager {
                                    td::Promise<td::Unit> promise) override {
     UNREACHABLE();
   }
+  void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                       std::function<td::Status(td::FileFd&)> write_data,
+                                       td::Promise<td::Unit> promise) override {
+    UNREACHABLE();
+  }
   void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise<td::Unit> promise) override {
     UNREACHABLE();
   }

validator/manager-init.cpp

@@ -227,7 +227,7 @@ void ValidatorManagerMasterchainReiniter::choose_masterchain_state() {
     }
     if (!p || ValidatorManager::is_persistent_state(h->unix_time(), p->unix_time())) {
       auto ttl = ValidatorManager::persistent_state_ttl(h->unix_time());
-      double time_to_download = 3600;
+      double time_to_download = 3600 * 3;
       if (ttl > td::Clocks::system() + time_to_download) {
         handle = h;
         break;
@@ -259,7 +259,7 @@ void ValidatorManagerMasterchainReiniter::download_masterchain_state() {
     }
   });
   td::actor::create_actor<DownloadShardState>("downloadstate", block_id_, block_id_, 2, manager_,
-                                              td::Timestamp::in(3600), std::move(P))
+                                              td::Timestamp::in(3600 * 3), std::move(P))
       .release();
 }
 
@@ -267,7 +267,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Ref<S
   state_ = td::Ref<MasterchainState>{std::move(state)};
   CHECK(handle_->received_state());
   CHECK(handle_->is_applied());
-
+  LOG(INFO) << "downloaded masterchain state";
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
     R.ensure();
     td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_all_shards);
@@ -276,6 +276,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Ref<S
 }
 
 void ValidatorManagerMasterchainReiniter::downloaded_all_shards() {
+  LOG(INFO) << "downloaded all shards";
   td::actor::send_closure(manager_, &ValidatorManager::update_gc_block_handle, handle_,
                           [SelfId = actor_id(this)](td::Result<td::Unit> R) {
                             R.ensure();
@@ -286,6 +287,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_all_shards() {
 void ValidatorManagerMasterchainReiniter::finish() {
   CHECK(handle_->id().id.seqno == 0 || handle_->is_key_block());
   promise_.set_value(ValidatorManagerInitResult{handle_, state_, std::move(client_), handle_, state_, handle_});
+  LOG(INFO) << "persistent state download finished";
   stop();
 }
 

validator/manager.cpp

@@ -366,17 +366,32 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) {
   if (!is_validator()) {
     return;
   }
+  if( ext_messages_.size() > max_mempool_num() ) {
+    return;
+  }
   auto R = create_ext_message(std::move(data));
   if (R.is_error()) {
     VLOG(VALIDATOR_NOTICE) << "dropping bad ihr message: " << R.move_as_error();
     return;
   }
-  auto M = std::make_unique<MessageExt<ExtMessage>>(R.move_as_ok());
+  add_external_message(R.move_as_ok());
-  auto id = M->ext_id();
+}
+
+void ValidatorManagerImpl::add_external_message(td::Ref<ExtMessage> msg) {
+  auto message = std::make_unique<MessageExt<ExtMessage>>(msg);
+  auto id = message->ext_id();
+  auto address = message->address();
+  unsigned long per_address_limit = 256;
+  if(ext_addr_messages_.count(address) < per_address_limit) {
     if (ext_messages_hashes_.count(id.hash) == 0) {
-    ext_messages_.emplace(id, std::move(M));
+      ext_messages_.emplace(id, std::move(message));
       ext_messages_hashes_.emplace(id.hash, id);
+      ext_addr_messages_[address].emplace(id.hash, id);
     }
+  }
+}
+void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise<td::Unit> promise) {
+  run_check_external_message(std::move(data), actor_id(this), std::move(promise));
 }
 
 void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) {
@@ -756,6 +771,7 @@ void ValidatorManagerImpl::get_external_messages(ShardIdFull shard,
       break;
     }
     if (it->second->expired()) {
+      ext_addr_messages_[it->second->address()].erase(it->first.hash);
       ext_messages_hashes_.erase(it->first.hash);
       it = ext_messages_.erase(it);
       continue;
@@ -804,17 +820,20 @@ void ValidatorManagerImpl::complete_external_messages(std::vector<ExtMessage::Ha
   for (auto &hash : to_delete) {
     auto it = ext_messages_hashes_.find(hash);
     if (it != ext_messages_hashes_.end()) {
+      ext_addr_messages_[ext_messages_[it->second]->address()].erase(it->first);
       CHECK(ext_messages_.erase(it->second));
       ext_messages_hashes_.erase(it);
     }
   }
+  unsigned long soft_mempool_limit = 1024;
   for (auto &hash : to_delay) {
     auto it = ext_messages_hashes_.find(hash);
     if (it != ext_messages_hashes_.end()) {
       auto it2 = ext_messages_.find(it->second);
-      if (it2->second->can_postpone()) {
+      if ((ext_messages_.size() < soft_mempool_limit) && it2->second->can_postpone()) {
         it2->second->postpone();
       } else {
+        ext_addr_messages_[it2->second->address()].erase(it2->first.hash);
         ext_messages_.erase(it2);
         ext_messages_hashes_.erase(it);
       }
@@ -1032,6 +1051,13 @@ void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, Bloc
                           std::move(promise));
 }
 
+void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                                           std::function<td::Status(td::FileFd&)> write_data,
+                                                           td::Promise<td::Unit> promise) {
+  td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id,
+                          std::move(write_data), std::move(promise));
+}
+
 void ValidatorManagerImpl::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state,
                                                  td::Promise<td::Unit> promise) {
   td::actor::send_closure(db_, &Db::store_zero_state_file, block_id, std::move(state), std::move(promise));
@@ -1328,7 +1354,7 @@ void ValidatorManagerImpl::send_get_zero_state_request(BlockIdExt id, td::uint32
 void ValidatorManagerImpl::send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id,
                                                              td::uint32 priority,
                                                              td::Promise<td::BufferSlice> promise) {
-  callback_->download_persistent_state(id, masterchain_block_id, priority, td::Timestamp::in(3600.0),
+  callback_->download_persistent_state(id, masterchain_block_id, priority, td::Timestamp::in(3600 * 3),
                                        std::move(promise));
 }
 
@@ -1349,6 +1375,7 @@ void ValidatorManagerImpl::send_get_next_key_blocks_request(BlockIdExt block_id,
 
 void ValidatorManagerImpl::send_external_message(td::Ref<ExtMessage> message) {
   callback_->send_ext_message(message->shard(), message->serialize());
+  add_external_message(std::move(message));
 }
 
 void ValidatorManagerImpl::send_ihr_message(td::Ref<IhrMessage> message) {

validator/manager.hpp

@@ -20,6 +20,8 @@
 
 #include "interfaces/validator-manager.h"
 #include "interfaces/db.h"
+#include "td/actor/PromiseFuture.h"
+#include "td/utils/port/Poll.h"
 #include "validator-group.hpp"
 #include "shard-client.hpp"
 #include "manager-init.h"
@@ -72,6 +74,9 @@ class MessageExt {
   auto hash() const {
     return message_->hash();
   }
+  auto address() const {
+    return std::make_pair(message_->wc(), message_->addr());
+  }
   bool is_active() {
     if (!active_) {
       if (reactivate_at_.is_in_past()) {
@@ -208,6 +213,7 @@ class ValidatorManagerImpl : public ValidatorManager {
   // DATA FOR COLLATOR
   std::map<ShardTopBlockDescriptionId, td::Ref<ShardTopBlockDescription>> shard_blocks_;
   std::map<MessageId<ExtMessage>, std::unique_ptr<MessageExt<ExtMessage>>> ext_messages_;
+  std::map<std::pair<ton::WorkchainId,ton::StdSmcAddress>, std::map<ExtMessage::Hash, MessageId<ExtMessage>>> ext_addr_messages_;
   std::map<ExtMessage::Hash, MessageId<ExtMessage>> ext_messages_hashes_;
   // IHR ?
   std::map<MessageId<IhrMessage>, std::unique_ptr<MessageExt<IhrMessage>>> ihr_messages_;
@@ -325,6 +331,9 @@ class ValidatorManagerImpl : public ValidatorManager {
   //void get_block_description(BlockIdExt block_id, td::Promise<BlockDescription> promise) override;
 
   void new_external_message(td::BufferSlice data) override;
+  void add_external_message(td::Ref<ExtMessage> message);
+  void check_external_message(td::BufferSlice data, td::Promise<td::Unit> promise) override;
+
   void new_ihr_message(td::BufferSlice data) override;
   void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override;
 
@@ -338,6 +347,9 @@ class ValidatorManagerImpl : public ValidatorManager {
                        td::Promise<td::Ref<ShardState>> promise) override;
   void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
                                    td::Promise<td::Unit> promise) override;
+  void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,
+                                       std::function<td::Status(td::FileFd&)> write_data,
+                                       td::Promise<td::Unit> promise) override;
   void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise<td::Unit> promise) override;
   void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout,
                         td::Promise<td::Ref<ShardState>> promise) override;
@@ -575,6 +587,9 @@ class ValidatorManagerImpl : public ValidatorManager {
   double block_ttl() const {
     return opts_->block_ttl();
   }
+  double max_mempool_num() const {
+    return opts_->max_mempool_num();
+  }
 
  private:
   std::map<BlockSeqno, WaitList<td::actor::Actor, td::Unit>> shard_client_waiters_;

validator/net/download-state.cpp

@@ -53,7 +53,7 @@ DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_i
 void DownloadState::abort_query(td::Status reason) {
   if (promise_) {
     if (reason.code() == ErrorCode::notready || reason.code() == ErrorCode::timeout) {
-      VLOG(FULL_NODE_DEBUG) << "failed to download state " << block_id_ << "from " << download_from_ << ": " << reason;
+      VLOG(FULL_NODE_DEBUG) << "failed to download state " << block_id_ << " from " << download_from_ << ": " << reason;
     } else {
       VLOG(FULL_NODE_NOTICE) << "failed to download state " << block_id_ << " from " << download_from_ << ": "
                              << reason;
@@ -115,6 +115,7 @@ void DownloadState::got_block_handle(BlockHandle handle) {
 
 void DownloadState::got_node_to_download(adnl::AdnlNodeIdShort node) {
   download_from_ = node;
+  LOG(INFO) << "downloading state " << block_id_ << " from " << download_from_;
 
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) mutable {
     if (R.is_error()) {
@@ -148,6 +149,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) {
     abort_query(F.move_as_error());
     return;
   }
+  prev_logged_timer_ = td::Timer();
 
   ton_api::downcast_call(
       *F.move_as_ok().get(),
@@ -187,6 +189,14 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques
   sum_ += data.size();
   parts_.push_back(std::move(data));
 
+  double elapsed = prev_logged_timer_.elapsed();
+  if (elapsed > 10.0) {
+    prev_logged_timer_ = td::Timer();
+    LOG(INFO) << "downloading state " << block_id_ << ": total=" << sum_ <<
+        " (" << double(sum_ - prev_logged_sum_) / elapsed << " B/s)";
+    prev_logged_sum_ = sum_;
+  }
+
   if (last_part) {
     td::BufferSlice res{td::narrow_cast<std::size_t>(sum_)};
     auto S = res.as_slice();
@@ -224,6 +234,7 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques
 
 void DownloadState::got_block_state(td::BufferSlice data) {
   state_ = std::move(data);
+  LOG(INFO) << "finished downloading state " << block_id_ << ": total=" << sum_;
   finish_query();
 }