integrating the existing state of TON Storage / TON Payments / CPS Fift development branches

CMakeLists.txt

@@ -232,7 +232,7 @@ elseif (CLANG OR GCC)
   if (APPLE)
     #use "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/export_list" for exported symbols
     set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S")
-    #set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S")
+    set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} -fvisibility=hidden -Wl,-dead_strip,-x,-S")
   else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections")
     set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,--exclude-libs,ALL")
@@ -378,12 +378,14 @@ add_subdirectory(tdfec)
 add_subdirectory(keyring)
 add_subdirectory(fec)
 add_subdirectory(rldp)
+add_subdirectory(rldp2)
 add_subdirectory(dht)
 add_subdirectory(overlay)
 add_subdirectory(catchain)
 add_subdirectory(validator-session)
 add_subdirectory(validator)
 add_subdirectory(blockchain-explorer)
+add_subdirectory(storage)
 add_subdirectory(validator-engine)
 add_subdirectory(validator-engine-console)
 add_subdirectory(create-hardfork)
@@ -445,7 +447,10 @@ endif()
 #BEGIN internal
 if (NOT TON_ONLY_TONLIB)
 add_executable(test-db test/test-td-main.cpp ${TONDB_TEST_SOURCE})
-target_link_libraries(test-db PRIVATE ton_db memprof)
+target_link_libraries(test-db PRIVATE ton_db memprof tdfec)
+
+add_executable(test-storage test/test-td-main.cpp ${STORAGE_TEST_SOURCE})
+target_link_libraries(test-storage PRIVATE storage ton_db memprof tl_api tl-utils fec rldp2)
 
 add_executable(test-rocksdb test/test-rocksdb.cpp)
 target_link_libraries(test-rocksdb PRIVATE memprof tddb tdutils)
@@ -469,6 +474,8 @@ add_executable(test-dht test/test-dht.cpp)
 target_link_libraries(test-dht adnl adnltest dht tl_api)
 add_executable(test-rldp test/test-rldp.cpp)
 target_link_libraries(test-rldp adnl adnltest dht rldp tl_api)
+add_executable(test-rldp2 test/test-rldp2.cpp)
+target_link_libraries(test-rldp2 adnl adnltest dht rldp2 tl_api)
 add_executable(test-validator-session-state test/test-validator-session-state.cpp)
 target_link_libraries(test-validator-session-state adnl dht rldp validatorsession tl_api)
 
@@ -536,6 +543,7 @@ if (NOT TON_ONLY_TONLIB)
 add_test(test-adnl test-adnl)
 add_test(test-dht test-dht)
 add_test(test-rldp test-rldp)
+add_test(test-rldp2 test-rldp2)
 #add_test(test-validator-session-state test-validator-session-state)
 add_test(test-catchain test-catchain)
 

adnl/adnl-peer.cpp

@@ -141,7 +141,8 @@ void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) {
     if (!packet.priority_addr_list().empty()) {
       update_addr_list(packet.priority_addr_list());
     }
-    VLOG(ADNL_NOTICE) << this << ": dropping IN message old our reinit date " << packet.reinit_date() << " date=" << d;
+    VLOG(ADNL_NOTICE) << this << ": dropping IN message old our reinit date " << packet.dst_reinit_date()
+                      << " date=" << d;
     auto M = OutboundAdnlMessage{adnlmessage::AdnlMessageNop{}, 0};
     send_message(std::move(M));
     return;

crypto/CMakeLists.txt

@@ -124,9 +124,6 @@ set(TON_CRYPTO_SOURCE
 
   vm/db/StaticBagOfCellsDb.h
   vm/db/StaticBagOfCellsDb.cpp
-
-  vm/db/BlobView.h
-  vm/db/BlobView.cpp
 )
 
 set(TON_DB_SOURCE
@@ -144,6 +141,7 @@ set(FIFT_SOURCE
   fift/Dictionary.cpp
   fift/Fift.cpp
   fift/IntCtx.cpp
+  fift/Continuation.cpp
   fift/SourceLookup.cpp
   fift/utils.cpp
   fift/words.cpp
@@ -151,6 +149,7 @@ set(FIFT_SOURCE
   fift/Dictionary.h
   fift/Fift.h
   fift/IntCtx.h
+  fift/Continuation.h
   fift/SourceLookup.h
   fift/utils.h
   fift/words.h
@@ -216,9 +215,7 @@ set(SMC_ENVELOPE_SOURCE
   smc-envelope/PaymentChannel.cpp
   smc-envelope/SmartContract.cpp
   smc-envelope/SmartContractCode.cpp
-  smc-envelope/TestGiver.cpp
-  smc-envelope/TestWallet.cpp
-  smc-envelope/Wallet.cpp
+  smc-envelope/WalletInterface.cpp
   smc-envelope/WalletV3.cpp
 
   smc-envelope/GenericAccount.h
@@ -228,9 +225,6 @@ set(SMC_ENVELOPE_SOURCE
   smc-envelope/MultisigWallet.h
   smc-envelope/SmartContract.h
   smc-envelope/SmartContractCode.h
-  smc-envelope/TestGiver.h
-  smc-envelope/TestWallet.h
-  smc-envelope/Wallet.h
   smc-envelope/WalletInterface.h
   smc-envelope/WalletV3.h
 )
@@ -268,7 +262,7 @@ set(FIFT_TEST_SOURCE
 add_library(ton_crypto STATIC ${TON_CRYPTO_SOURCE})
 target_include_directories(ton_crypto PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
 	$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
-target_link_libraries(ton_crypto PUBLIC ${OPENSSL_CRYPTO_LIBRARY} tdutils)
+target_link_libraries(ton_crypto PUBLIC ${OPENSSL_CRYPTO_LIBRARY} tdutils tddb_utils)
 if (NOT WIN32)
   target_link_libraries(ton_crypto PUBLIC dl z)
 endif()

crypto/common/refcnt.cpp

@@ -21,6 +21,11 @@
 #include "td/utils/ScopeGuard.h"
 
 namespace td {
+
+Ref<CntObject> CntObject::clone() const {
+  return Ref<CntObject>{make_copy(), Ref<CntObject>::acquire_t()};
+}
+
 namespace detail {
 struct SafeDeleter {
  public:

crypto/common/refcnt.hpp

@@ -83,6 +83,7 @@ class CntObject {
   void assert_unique() const {
     assert(is_unique());
   }
+  Ref<CntObject> clone() const;
 };
 
 typedef Ref<CntObject> RefAny;
@@ -159,6 +160,7 @@ struct static_cast_ref {};
 namespace detail {
 void safe_delete(const CntObject* ptr);
 }
+
 template <class T>
 class Ref {
   T* ptr;

crypto/fift/Continuation.cpp

@@ -0,0 +1,398 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2020 Telegram Systems LLP
+*/
+#include "Continuation.h"
+#include "IntCtx.h"
+#include "Dictionary.h"
+
+namespace fift {
+
+//
+// FiftCont
+//
+bool FiftCont::print_dict_name(std::ostream& os, const IntCtx& ctx) const {
+  std::string word_name;
+  if (ctx.dictionary && ctx.dictionary->lookup_def(this, &word_name)) {
+    if (word_name.size() && word_name.back() == ' ') {
+      word_name.pop_back();
+    }
+    os << word_name;
+    return true;
+  }
+  return false;
+}
+
+std::string FiftCont::get_dict_name(const IntCtx& ctx) const {
+  std::string word_name;
+  if (ctx.dictionary && ctx.dictionary->lookup_def(this, &word_name)) {
+    if (word_name.size() && word_name.back() == ' ') {
+      word_name.pop_back();
+    }
+    return word_name;
+  }
+  return {};
+}
+
+bool FiftCont::print_dummy_name(std::ostream& os, const IntCtx& ctx) const {
+  os << "<continuation " << (const void*)this << ">";
+  return false;
+}
+
+bool FiftCont::print_name(std::ostream& os, const IntCtx& ctx) const {
+  return print_dict_name(os, ctx) || print_dummy_name(os, ctx);
+}
+
+bool FiftCont::dump(std::ostream& os, const IntCtx& ctx) const {
+  bool ok = print_name(os, ctx);
+  os << std::endl;
+  return ok;
+}
+
+//
+// QuitCont
+//
+Ref<FiftCont> QuitCont::run_tail(IntCtx& ctx) const {
+  ctx.set_exit_code(exit_code);
+  ctx.next.clear();
+  return {};
+}
+
+bool QuitCont::print_name(std::ostream& os, const IntCtx& ctx) const {
+  os << "<quit " << exit_code << ">";
+  return true;
+}
+
+//
+// SeqCont
+//
+Ref<FiftCont> SeqCont::run_tail(IntCtx& ctx) const {
+  ctx.next = seq(second, std::move(ctx.next));
+  return first;
+}
+
+Ref<FiftCont> SeqCont::run_modify(IntCtx& ctx) {
+  if (ctx.next.is_null()) {
+    ctx.next = std::move(second);
+    return std::move(first);
+  } else {
+    auto res = std::move(first);
+    first = std::move(second);
+    second = std::move(ctx.next);
+    ctx.next = self();
+    return res;
+  }
+}
+
+bool SeqCont::print_name(std::ostream& os, const IntCtx& ctx) const {
+  if (first.not_null()) {
+    return first->print_name(os, ctx);
+  } else {
+    return true;
+  }
+}
+
+bool SeqCont::dump(std::ostream& os, const IntCtx& ctx) const {
+  if (first.not_null()) {
+    return first->dump(os, ctx);
+  } else {
+    return true;
+  }
+}
+
+//
+// TimesCont
+//
+Ref<FiftCont> TimesCont::run_tail(IntCtx& ctx) const {
+  if (count > 1) {
+    ctx.next = td::make_ref<TimesCont>(body, SeqCont::seq(after, std::move(ctx.next)), count - 1);
+  } else {
+    ctx.next = SeqCont::seq(after, std::move(ctx.next));
+  }
+  return body;
+}
+
+Ref<FiftCont> TimesCont::run_modify(IntCtx& ctx) {
+  if (ctx.next.not_null()) {
+    after = SeqCont::seq(std::move(after), std::move(ctx.next));
+  }
+  if (count > 1) {
+    --count;
+    ctx.next = self();
+    return body;
+  } else {
+    ctx.next = std::move(after);
+    return std::move(body);
+  }
+}
+
+bool TimesCont::print_name(std::ostream& os, const IntCtx& ctx) const {
+  os << "<repeat " << count << " times>";
+  return true;
+}
+
+bool TimesCont::dump(std::ostream& os, const IntCtx& ctx) const {
+  os << "<repeat " << count << " times:> ";
+  return body->dump(os, ctx);
+}
+
+//
+// UntilCont
+//
+Ref<FiftCont> UntilCont::run_tail(IntCtx& ctx) const {
+  if (ctx.stack.pop_bool()) {
+    return after;
+  } else if (ctx.next.not_null()) {
+    ctx.next = td::make_ref<UntilCont>(body, SeqCont::seq(after, std::move(ctx.next)));
+    return body;
+  } else {
+    ctx.next = self();
+    return body;
+  }
+}
+
+Ref<FiftCont> UntilCont::run_modify(IntCtx& ctx) {
+  if (ctx.stack.pop_bool()) {
+    return std::move(after);
+  } else {
+    if (ctx.next.not_null()) {
+      after = SeqCont::seq(after, std::move(ctx.next));
+    }
+    ctx.next = self();
+    return body;
+  }
+}
+
+bool UntilCont::print_name(std::ostream& os, const IntCtx& ctx) const {
+  os << "<until loop continuation>";
+  return true;
+}
+
+bool UntilCont::dump(std::ostream& os, const IntCtx& ctx) const {
+  os << "<until loop continuation:> ";
+  return body->dump(os, ctx);
+}
+
+//
+// WhileCont
+//
+Ref<FiftCont> WhileCont::run_tail(IntCtx& ctx) const {
+  if (!stage) {
+    ctx.next = td::make_ref<WhileCont>(cond, body, SeqCont::seq(after, std::move(ctx.next)), true);
+    return cond;
+  }
+  if (!ctx.stack.pop_bool()) {
+    return after;
+  } else {
+    ctx.next = td::make_ref<WhileCont>(cond, body, SeqCont::seq(after, std::move(ctx.next)));
+    return body;
+  }
+}
+
+Ref<FiftCont> WhileCont::run_modify(IntCtx& ctx) {
+  if (!stage) {
+    if (ctx.next.not_null()) {
+      after = SeqCont::seq(std::move(after), std::move(ctx.next));
+    }
+    stage = true;
+    ctx.next = self();
+    return cond;
+  }
+  if (!ctx.stack.pop_bool()) {
+    return std::move(after);
+  } else {
+    if (ctx.next.not_null()) {
+      after = SeqCont::seq(std::move(after), std::move(ctx.next));
+    }
+    stage = false;
+    ctx.next = self();
+    return body;
+  }
+}
+
+bool WhileCont::print_name(std::ostream& os, const IntCtx& ctx) const {
+  os << "<while loop " << (stage ? "body" : "condition") << ">";
+  return true;
+}
+
+bool WhileCont::dump(std::ostream& os, const IntCtx& ctx) const {
+  os << "<while loop " << (stage ? "body" : "condition") << ":> ";
+  return (stage ? body : cond)->dump(os, ctx);
+}
+
+//
+// LoopCont
+//
+Ref<FiftCont> LoopCont::run_tail(IntCtx& ctx) const {
+  return Ref<FiftCont>(clone());
+}
+
+Ref<FiftCont> LoopCont::run_modify(IntCtx& ctx) {
+  if (ctx.next.not_null()) {
+    after = SeqCont::seq(std::move(after), std::move(ctx.next));
+  }
+  switch (state) {
+    case 0:
+      if (!init(ctx)) {
+        return std::move(after);
+      }
+      state = 1;
+      // fallthrough
+    case 1:
+      if (!pre_exec(ctx)) {
+        state = 3;
+        if (finalize(ctx)) {
+          return std::move(after);
+        } else {
+          return {};
+        }
+      }
+      state = 2;
+      ctx.next = self();
+      return func;
+    case 2:
+      if (post_exec(ctx)) {
+        state = 1;
+        return self();
+      }
+      state = 3;
+      // fallthrough
+    case 3:
+      if (finalize(ctx)) {
+        return std::move(after);
+      } else {
+        return {};
+      }
+    default:
+      throw IntError{"invalid LoopCont state"};
+  }
+}
+
+bool LoopCont::print_name(std::ostream& os, const IntCtx& ctx) const {
+  os << "<generic loop continuation state " << state << ">";
+  return true;
+}
+
+//
+// GenericLitCont
+//
+bool GenericLitCont::print_name(std::ostream& os, const IntCtx& ctx) const {
+  auto list = get_literals();
+  bool sp = false;
+  for (auto entry : list) {
+    if (sp) {
+      os << sp;
+    }
+    sp = true;
+    int tp = entry.type();
+    if (entry.is_int() || entry.is(vm::StackEntry::t_string) || entry.is(vm::StackEntry::t_bytes)) {
+      entry.dump(os);
+    } else {
+      auto cont_lit = entry.as_object<FiftCont>();
+      if (cont_lit.not_null()) {
+        os << "{ ";
+        cont_lit->print_name(os, ctx);
+        os << " }";
+      } else {
+        os << "<literal of type " << tp << ">";
+      }
+    }
+  }
+  return true;
+}
+
+//
+// SmallIntLitCont
+//
+Ref<FiftCont> SmallIntLitCont::run_tail(IntCtx& ctx) const {
+  ctx.stack.push_smallint(value_);
+  return {};
+}
+
+std::vector<vm::StackEntry> SmallIntLitCont::get_literals() const {
+  return {td::make_refint(value_)};
+}
+
+//
+// IntLitCont
+//
+Ref<FiftCont> IntLitCont::run_tail(IntCtx& ctx) const {
+  ctx.stack.push_int(value_);
+  return {};
+}
+
+Ref<FiftCont> IntLitCont::run_modify(IntCtx& ctx) {
+  ctx.stack.push_int(std::move(value_));
+  return {};
+}
+
+Ref<FiftCont> IntLitCont::literal(td::RefInt256 int_value) {
+  if (int_value->signed_fits_bits(64)) {
+    return literal(int_value->to_long());
+  } else {
+    return td::make_ref<IntLitCont>(std::move(int_value));
+  }
+}
+
+//
+// LitCont
+//
+Ref<FiftCont> LitCont::run_tail(IntCtx& ctx) const {
+  ctx.stack.push(value_);
+  return {};
+}
+
+Ref<FiftCont> LitCont::run_modify(IntCtx& ctx) {
+  ctx.stack.push(std::move(value_));
+  return {};
+}
+
+Ref<FiftCont> LitCont::literal(vm::StackEntry value) {
+  if (value.is_int()) {
+    return literal(std::move(value).as_int());
+  } else {
+    return td::make_ref<LitCont>(std::move(value));
+  }
+}
+
+//
+// MultiLitCont
+//
+Ref<FiftCont> MultiLitCont::run_tail(IntCtx& ctx) const {
+  for (auto& value : values_) {
+    ctx.stack.push(value);
+  }
+  return {};
+}
+
+Ref<FiftCont> MultiLitCont::run_modify(IntCtx& ctx) {
+  for (auto& value : values_) {
+    ctx.stack.push(std::move(value));
+  }
+  return {};
+}
+
+MultiLitCont& MultiLitCont::push_back(vm::StackEntry new_literal) {
+  values_.push_back(std::move(new_literal));
+  return *this;
+}
+
+vm::StackEntry MultiLitCont::at(int idx) const {
+  return values_.at(idx);
+}
+
+}  // namespace fift

crypto/fift/Continuation.h

@@ -0,0 +1,272 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2020 Telegram Systems LLP
+*/
+#pragma once
+#include "common/refcnt.hpp"
+#include "common/refint.h"
+#include "vm/stack.hpp"
+
+namespace fift {
+using td::Ref;
+struct IntCtx;
+
+/*
+ * 
+ *    FIFT CONTINUATIONS
+ * 
+ */
+
+class FiftCont : public td::CntObject {
+ public:
+  FiftCont() = default;
+  virtual ~FiftCont() override = default;
+  virtual Ref<FiftCont> run_tail(IntCtx& ctx) const = 0;
+  virtual Ref<FiftCont> run_modify(IntCtx& ctx) {
+    return run_tail(ctx);
+  }
+  virtual Ref<FiftCont> handle_tail(IntCtx& ctx) const {
+    return {};
+  }
+  virtual Ref<FiftCont> handle_modify(IntCtx& ctx) {
+    return handle_tail(ctx);
+  }
+  virtual Ref<FiftCont> up() const {
+    return {};
+  }
+  virtual bool is_list() const {
+    return false;
+  }
+  virtual long long list_size() const {
+    return -1;
+  }
+  virtual const Ref<FiftCont>* get_list() const {
+    return nullptr;
+  }
+  virtual bool is_literal() const {
+    return false;
+  }
+  virtual int literal_count() const {
+    return -1;
+  }
+  virtual std::vector<vm::StackEntry> get_literals() const {
+    return {};
+  }
+  std::string get_dict_name(const IntCtx& ctx) const;
+  bool print_dict_name(std::ostream& os, const IntCtx& ctx) const;
+  bool print_dummy_name(std::ostream& os, const IntCtx& ctx) const;
+  virtual bool print_name(std::ostream& os, const IntCtx& ctx) const;
+  virtual bool dump(std::ostream& os, const IntCtx& ctx) const;
+  Ref<FiftCont> self() const {
+    return Ref<FiftCont>{this};
+  }
+};
+
+class QuitCont : public FiftCont {
+  int exit_code;
+
+ public:
+  QuitCont(int _exit_code = -1) : exit_code(_exit_code) {
+  }
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
+  bool print_name(std::ostream& os, const IntCtx& ctx) const override;
+};
+
+class SeqCont : public FiftCont {
+  Ref<FiftCont> first, second;
+
+ public:
+  SeqCont(Ref<FiftCont> _first, Ref<FiftCont> _second) : first(std::move(_first)), second(std::move(_second)) {
+  }
+  static Ref<FiftCont> seq(Ref<FiftCont> _first, Ref<FiftCont> _second) {
+    return _second.is_null() ? std::move(_first) : td::make_ref<SeqCont>(std::move(_first), std::move(_second));
+  }
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_modify(IntCtx& ctx) override;
+  Ref<FiftCont> up() const override {
+    return second;
+  }
+  bool print_name(std::ostream& os, const IntCtx& ctx) const override;
+  bool dump(std::ostream& os, const IntCtx& ctx) const override;
+};
+
+class TimesCont : public FiftCont {
+  Ref<FiftCont> body, after;
+  int count;
+
+ public:
+  TimesCont(Ref<FiftCont> _body, Ref<FiftCont> _after, int _count)
+      : body(std::move(_body)), after(std::move(_after)), count(_count) {
+  }
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_modify(IntCtx& ctx) override;
+  Ref<FiftCont> up() const override {
+    return after;
+  }
+  bool print_name(std::ostream& os, const IntCtx& ctx) const override;
+  bool dump(std::ostream& os, const IntCtx& ctx) const override;
+};
+
+class UntilCont : public FiftCont {
+  Ref<FiftCont> body, after;
+
+ public:
+  UntilCont(Ref<FiftCont> _body, Ref<FiftCont> _after) : body(std::move(_body)), after(std::move(_after)) {
+  }
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_modify(IntCtx& ctx) override;
+  bool print_name(std::ostream& os, const IntCtx& ctx) const override;
+  bool dump(std::ostream& os, const IntCtx& ctx) const override;
+};
+
+class WhileCont : public FiftCont {
+  Ref<FiftCont> cond, body, after;
+  bool stage;
+
+ public:
+  WhileCont(Ref<FiftCont> _cond, Ref<FiftCont> _body, Ref<FiftCont> _after, bool _stage = false)
+      : cond(std::move(_cond)), body(std::move(_body)), after(std::move(_after)), stage(_stage) {
+  }
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_modify(IntCtx& ctx) override;
+  Ref<FiftCont> up() const override {
+    return after;
+  }
+  bool print_name(std::ostream& os, const IntCtx& ctx) const override;
+  bool dump(std::ostream& os, const IntCtx& ctx) const override;
+};
+
+class LoopCont : public FiftCont {
+  Ref<FiftCont> func, after;
+  int state;
+
+ public:
+  LoopCont(Ref<FiftCont> _func, Ref<FiftCont> _after, int _state = 0)
+      : func(std::move(_func)), after(std::move(_after)), state(_state) {
+  }
+  LoopCont(const LoopCont&) = default;
+  virtual bool init(IntCtx& ctx) {
+    return true;
+  }
+  virtual bool pre_exec(IntCtx& ctx) {
+    return true;
+  }
+  virtual bool post_exec(IntCtx& ctx) {
+    return true;
+  }
+  virtual bool finalize(IntCtx& ctx) {
+    return true;
+  }
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_modify(IntCtx& ctx) override;
+  Ref<FiftCont> up() const override {
+    return after;
+  }
+  bool print_name(std::ostream& os, const IntCtx& ctx) const override;
+};
+
+class GenericLitCont : public FiftCont {
+ public:
+  bool is_literal() const override {
+    return true;
+  }
+  std::vector<vm::StackEntry> get_literals() const override = 0;
+  bool print_name(std::ostream& os, const IntCtx& ctx) const override;
+};
+
+class SmallIntLitCont : public GenericLitCont {
+  long long value_;
+
+ public:
+  SmallIntLitCont(long long value) : value_(value) {
+  }
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
+  std::vector<vm::StackEntry> get_literals() const override;
+  static Ref<FiftCont> literal(long long int_value) {
+    return td::make_ref<SmallIntLitCont>(int_value);
+  }
+};
+
+class IntLitCont : public GenericLitCont {
+  td::RefInt256 value_;
+
+ public:
+  IntLitCont(td::RefInt256 value) : value_(std::move(value)) {
+  }
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_modify(IntCtx& ctx) override;
+  std::vector<vm::StackEntry> get_literals() const override {
+    return {vm::StackEntry(value_)};
+  }
+  static Ref<FiftCont> literal(td::RefInt256 int_value);
+  static Ref<FiftCont> literal(long long int_value) {
+    return SmallIntLitCont::literal(int_value);
+  }
+};
+
+class LitCont : public GenericLitCont {
+  vm::StackEntry value_;
+
+ public:
+  LitCont(const vm::StackEntry& value) : value_(value) {
+  }
+  LitCont(vm::StackEntry&& value) : value_(std::move(value)) {
+  }
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_modify(IntCtx& ctx) override;
+  std::vector<vm::StackEntry> get_literals() const override {
+    return {value_};
+  }
+  static Ref<FiftCont> literal(vm::StackEntry value);
+  static Ref<FiftCont> literal(td::RefInt256 int_value) {
+    return IntLitCont::literal(std::move(int_value));
+  }
+  static Ref<FiftCont> literal(long long int_value) {
+    return SmallIntLitCont::literal(int_value);
+  }
+};
+
+class MultiLitCont : public GenericLitCont {
+  std::vector<vm::StackEntry> values_;
+
+ public:
+  MultiLitCont(const std::vector<vm::StackEntry>& values) : values_(values) {
+  }
+  MultiLitCont(std::vector<vm::StackEntry>&& values) : values_(std::move(values)) {
+  }
+  MultiLitCont(std::initializer_list<vm::StackEntry> value_list) : values_(value_list) {
+  }
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_modify(IntCtx& ctx) override;
+  std::vector<vm::StackEntry> get_literals() const override {
+    return values_;
+  }
+  MultiLitCont& push_back(vm::StackEntry new_literal);
+  vm::StackEntry at(int idx) const;
+};
+
+class InterpretCont : public FiftCont {
+ public:
+  InterpretCont() = default;
+  Ref<FiftCont> run_tail(IntCtx&) const override;  // text interpreter, defined in words.cpp
+  bool print_name(std::ostream& os, const IntCtx& ctx) const override {
+    os << "<text interpreter continuation>";
+    return true;
+  }
+};
+
+}  // namespace fift

crypto/fift/Dictionary.cpp

@@ -20,20 +20,10 @@
 
 namespace fift {
 
-//
-// WordDef
-//
-void WordDef::run(IntCtx& ctx) const {
-  auto next = run_tail(ctx);
-  while (next.not_null()) {
-    next = next->run_tail(ctx);
-  }
-}
-
 //
 // StackWord
 //
-Ref<WordDef> StackWord::run_tail(IntCtx& ctx) const {
+Ref<FiftCont> StackWord::run_tail(IntCtx& ctx) const {
   f(ctx.stack);
   return {};
 }
@@ -41,7 +31,7 @@ Ref<WordDef> StackWord::run_tail(IntCtx& ctx) const {
 //
 // CtxWord
 //
-Ref<WordDef> CtxWord::run_tail(IntCtx& ctx) const {
+Ref<FiftCont> CtxWord::run_tail(IntCtx& ctx) const {
   f(ctx);
   return {};
 }
@@ -49,57 +39,125 @@ Ref<WordDef> CtxWord::run_tail(IntCtx& ctx) const {
 //
 // CtxTailWord
 //
-Ref<WordDef> CtxTailWord::run_tail(IntCtx& ctx) const {
+Ref<FiftCont> CtxTailWord::run_tail(IntCtx& ctx) const {
   return f(ctx);
 }
 
 //
 // WordList
 //
-WordList::WordList(std::vector<Ref<WordDef>>&& _list) : list(std::move(_list)) {
+WordList::WordList(std::vector<Ref<FiftCont>>&& _list) : list(std::move(_list)) {
 }
 
-WordList::WordList(const std::vector<Ref<WordDef>>& _list) : list(_list) {
+WordList::WordList(const std::vector<Ref<FiftCont>>& _list) : list(_list) {
 }
 
-WordList& WordList::push_back(Ref<WordDef> word_def) {
+WordList& WordList::push_back(Ref<FiftCont> word_def) {
   list.push_back(std::move(word_def));
   return *this;
 }
 
-WordList& WordList::push_back(WordDef& wd) {
+WordList& WordList::push_back(FiftCont& wd) {
   list.emplace_back(&wd);
   return *this;
 }
 
-Ref<WordDef> WordList::run_tail(IntCtx& ctx) const {
+Ref<FiftCont> WordList::run_tail(IntCtx& ctx) const {
   if (list.empty()) {
     return {};
   }
-  auto it = list.cbegin(), it2 = list.cend() - 1;
-  while (it < it2) {
-    (*it)->run(ctx);
-    ++it;
+  if (list.size() > 1) {
+    ctx.next = td::make_ref<ListCont>(std::move(ctx.next), Ref<WordList>(this), 1);
   }
-  return *it;
+  return list[0];
 }
 
 void WordList::close() {
   list.shrink_to_fit();
 }
 
-WordList& WordList::append(const std::vector<Ref<WordDef>>& other) {
+WordList& WordList::append(const std::vector<Ref<FiftCont>>& other) {
   list.insert(list.end(), other.begin(), other.end());
   return *this;
 }
 
+WordList& WordList::append(const Ref<FiftCont>* begin, const Ref<FiftCont>* end) {
+  list.insert(list.end(), begin, end);
+  return *this;
+}
+
+bool WordList::dump(std::ostream& os, const IntCtx& ctx) const {
+  os << "{";
+  for (auto entry : list) {
+    os << ' ';
+    entry->print_name(os, ctx);
+  }
+  os << " }" << std::endl;
+  return true;
+}
+
+//
+// ListCont
+//
+
+Ref<FiftCont> ListCont::run_tail(IntCtx& ctx) const {
+  auto sz = list->size();
+  if (pos >= sz) {
+    return std::move(ctx.next);
+  } else if (ctx.next.not_null()) {
+    ctx.next = td::make_ref<ListCont>(SeqCont::seq(next, std::move(ctx.next)), list, pos + 1);
+  } else if (pos + 1 == sz) {
+    ctx.next = next;
+  } else {
+    ctx.next = td::make_ref<ListCont>(next, list, pos + 1);
+  }
+  return list->at(pos);
+}
+
+Ref<FiftCont> ListCont::run_modify(IntCtx& ctx) {
+  auto sz = list->size();
+  if (pos >= sz) {
+    return std::move(ctx.next);
+  }
+  auto cur = list->at(pos++);
+  if (ctx.next.not_null()) {
+    next = SeqCont::seq(next, std::move(ctx.next));
+  }
+  if (pos == sz) {
+    ctx.next = std::move(next);
+  } else {
+    ctx.next = self();
+  }
+  return cur;
+}
+
+bool ListCont::dump(std::ostream& os, const IntCtx& ctx) const {
+  std::string dict_name = list->get_dict_name(ctx);
+  if (!dict_name.empty()) {
+    os << "[in " << dict_name << ":] ";
+  }
+  std::size_t sz = list->size(), i, a = (pos >= 16 ? pos - 16 : 0), b = std::min(pos + 16, sz);
+  if (a > 0) {
+    os << "... ";
+  }
+  for (i = a; i < b; i++) {
+    if (i == pos) {
+      os << "**HERE** ";
+    }
+    list->at(i)->print_name(os, ctx);
+    os << ' ';
+  }
+  if (b < sz) {
+    os << "...";
+  }
+  os << std::endl;
+  return true;
+}
+
 //
 // DictEntry
 //
 
-DictEntry::DictEntry(Ref<WordDef> _def, bool _act) : def(std::move(_def)), active(_act) {
-}
-
 DictEntry::DictEntry(StackWordFunc func) : def(Ref<StackWord>{true, std::move(func)}), active(false) {
 }
 
@@ -109,22 +167,6 @@ DictEntry::DictEntry(CtxWordFunc func, bool _act) : def(Ref<CtxWord>{true, std::
 DictEntry::DictEntry(CtxTailWordFunc func, bool _act) : def(Ref<CtxTailWord>{true, std::move(func)}), active(_act) {
 }
 
-Ref<WordDef> DictEntry::get_def() const& {
-  return def;
-}
-
-Ref<WordDef> DictEntry::get_def() && {
-  return std::move(def);
-}
-
-void DictEntry::operator()(IntCtx& ctx) const {
-  def->run(ctx);
-}
-
-bool DictEntry::is_active() const {
-  return active;
-}
-
 //
 // Dictionary
 //
@@ -141,7 +183,7 @@ void Dictionary::def_ctx_word(std::string name, CtxWordFunc func) {
 }
 
 void Dictionary::def_active_word(std::string name, CtxWordFunc func) {
-  Ref<WordDef> wdef = Ref<CtxWord>{true, std::move(func)};
+  Ref<FiftCont> wdef = Ref<CtxWord>{true, std::move(func)};
   def_word(std::move(name), {std::move(wdef), true});
 }
 
@@ -166,17 +208,32 @@ void Dictionary::undef_word(td::Slice name) {
   words_.erase(it);
 }
 
+bool Dictionary::lookup_def(const FiftCont* cont, std::string* word_ptr) const {
+  if (!cont) {
+    return false;
+  }
+  for (const auto& entry : words_) {
+    if (entry.second.get_def().get() == cont) {
+      if (word_ptr) {
+        *word_ptr = entry.first;
+      }
+      return true;
+    }
+  }
+  return false;
+}
+
 void interpret_nop(vm::Stack& stack) {
 }
 
-Ref<WordDef> Dictionary::nop_word_def = Ref<StackWord>{true, interpret_nop};
+Ref<FiftCont> Dictionary::nop_word_def = Ref<StackWord>{true, interpret_nop};
 
 //
 // functions for wordef
 //
-Ref<WordDef> pop_exec_token(vm::Stack& stack) {
+Ref<FiftCont> pop_exec_token(vm::Stack& stack) {
   stack.check_underflow(1);
-  auto wd_ref = stack.pop().as_object<WordDef>();
+  auto wd_ref = stack.pop().as_object<FiftCont>();
   if (wd_ref.is_null()) {
     throw IntError{"execution token expected"};
   }

crypto/fift/Dictionary.h

@@ -22,9 +22,11 @@
 #include <map>
 
 #include "IntCtx.h"
+#include "Continuation.h"
 
 namespace fift {
 using td::Ref;
+
 /*
  *
  *    WORD CLASSES
@@ -34,66 +36,49 @@ using td::Ref;
 typedef std::function<void(vm::Stack&)> StackWordFunc;
 typedef std::function<void(IntCtx&)> CtxWordFunc;
 
-class WordDef : public td::CntObject {
- public:
-  WordDef() = default;
-  virtual ~WordDef() override = default;
-  virtual Ref<WordDef> run_tail(IntCtx& ctx) const = 0;
-  void run(IntCtx& ctx) const;
-  virtual bool is_list() const {
-    return false;
-  }
-  virtual long long list_size() const {
-    return -1;
-  }
-  virtual const std::vector<Ref<WordDef>>* get_list() const {
-    return nullptr;
-  }
-};
-
-class StackWord : public WordDef {
+class StackWord : public FiftCont {
   StackWordFunc f;
 
  public:
   StackWord(StackWordFunc _f) : f(std::move(_f)) {
   }
   ~StackWord() override = default;
-  Ref<WordDef> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
 };
 
-class CtxWord : public WordDef {
+class CtxWord : public FiftCont {
   CtxWordFunc f;
 
  public:
   CtxWord(CtxWordFunc _f) : f(std::move(_f)) {
   }
   ~CtxWord() override = default;
-  Ref<WordDef> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
 };
 
-typedef std::function<Ref<WordDef>(IntCtx&)> CtxTailWordFunc;
+typedef std::function<Ref<FiftCont>(IntCtx&)> CtxTailWordFunc;
 
-class CtxTailWord : public WordDef {
+class CtxTailWord : public FiftCont {
   CtxTailWordFunc f;
 
  public:
   CtxTailWord(CtxTailWordFunc _f) : f(std::move(_f)) {
   }
   ~CtxTailWord() override = default;
-  Ref<WordDef> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
 };
 
-class WordList : public WordDef {
-  std::vector<Ref<WordDef>> list;
+class WordList : public FiftCont {
+  std::vector<Ref<FiftCont>> list;
 
  public:
   ~WordList() override = default;
   WordList() = default;
-  WordList(std::vector<Ref<WordDef>>&& _list);
-  WordList(const std::vector<Ref<WordDef>>& _list);
-  WordList& push_back(Ref<WordDef> word_def);
-  WordList& push_back(WordDef& wd);
-  Ref<WordDef> run_tail(IntCtx& ctx) const override;
+  WordList(std::vector<Ref<FiftCont>>&& _list);
+  WordList(const std::vector<Ref<FiftCont>>& _list);
+  WordList& push_back(Ref<FiftCont> word_def);
+  WordList& push_back(FiftCont& wd);
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
   void close();
   bool is_list() const override {
     return true;
@@ -101,43 +86,73 @@ class WordList : public WordDef {
   long long list_size() const override {
     return (long long)list.size();
   }
-  const std::vector<Ref<WordDef>>* get_list() const override {
-    return &list;
+  std::size_t size() const {
+    return list.size();
   }
-  WordList& append(const std::vector<Ref<WordDef>>& other);
+  const Ref<FiftCont>& at(std::size_t idx) const {
+    return list.at(idx);
+  }
+  const Ref<FiftCont>* get_list() const override {
+    return list.data();
+  }
+  WordList& append(const std::vector<Ref<FiftCont>>& other);
+  WordList& append(const Ref<FiftCont>* begin, const Ref<FiftCont>* end);
   WordList* make_copy() const override {
     return new WordList(list);
   }
+  bool dump(std::ostream& os, const IntCtx& ctx) const override;
+};
+
+class ListCont : public FiftCont {
+  Ref<FiftCont> next;
+  Ref<WordList> list;
+  std::size_t pos;
+
+ public:
+  ListCont(Ref<FiftCont> nxt, Ref<WordList> wl, std::size_t p = 0) : next(std::move(nxt)), list(std::move(wl)), pos(p) {
+  }
+  ~ListCont() override = default;
+  Ref<FiftCont> run_tail(IntCtx& ctx) const override;
+  Ref<FiftCont> run_modify(IntCtx& ctx) override;
+  Ref<FiftCont> up() const override {
+    return next;
+  }
+  bool dump(std::ostream& os, const IntCtx& ctx) const override;
 };
 
 class DictEntry {
-  Ref<WordDef> def;
+  Ref<FiftCont> def;
   bool active;
 
  public:
   DictEntry() = delete;
   DictEntry(const DictEntry& ref) = default;
   DictEntry(DictEntry&& ref) = default;
-  DictEntry(Ref<WordDef> _def, bool _act = false);
+  DictEntry(Ref<FiftCont> _def, bool _act = false) : def(std::move(_def)), active(_act) {
+  }
   DictEntry(StackWordFunc func);
   DictEntry(CtxWordFunc func, bool _act = false);
   DictEntry(CtxTailWordFunc func, bool _act = false);
-  //DictEntry(const std::vector<Ref<WordDef>>& word_list);
-  //DictEntry(std::vector<Ref<WordDef>>&& word_list);
+  //DictEntry(const std::vector<Ref<FiftCont>>& word_list);
+  //DictEntry(std::vector<Ref<FiftCont>>&& word_list);
   DictEntry& operator=(const DictEntry&) = default;
   DictEntry& operator=(DictEntry&&) = default;
-  Ref<WordDef> get_def() const&;
-  Ref<WordDef> get_def() &&;
-  void operator()(IntCtx& ctx) const;
-  bool is_active() const;
-  ~DictEntry() = default;
+  Ref<FiftCont> get_def() const& {
+    return def;
+  }
+  Ref<FiftCont> get_def() && {
+    return std::move(def);
+  }
+  bool is_active() const {
+    return active;
+  }
 };
 
 /*
-DictEntry::DictEntry(const std::vector<Ref<WordDef>>& word_list) : def(Ref<WordList>{true, word_list}) {
+DictEntry::DictEntry(const std::vector<Ref<FiftCont>>& word_list) : def(Ref<WordList>{true, word_list}) {
 }
 
-DictEntry::DictEntry(std::vector<Ref<WordDef>>&& word_list) : def(Ref<WordList>{true, std::move(word_list)}) {
+DictEntry::DictEntry(std::vector<Ref<FiftCont>>&& word_list) : def(Ref<WordList>{true, std::move(word_list)}) {
 }
 */
 
@@ -156,7 +171,10 @@ class Dictionary {
   void def_stack_word(std::string name, StackWordFunc func);
   void def_word(std::string name, DictEntry word);
   void undef_word(td::Slice name);
-
+  bool lookup_def(const FiftCont* cont, std::string* word_ptr = nullptr) const;
+  bool lookup_def(Ref<FiftCont> cont, std::string* word_ptr = nullptr) const {
+    return lookup_def(cont.get(), word_ptr);
+  }
   auto begin() const {
     return words_.begin();
   }
@@ -164,7 +182,7 @@ class Dictionary {
     return words_.end();
   }
 
-  static Ref<WordDef> nop_word_def;
+  static Ref<FiftCont> nop_word_def;
 
  private:
   std::map<std::string, DictEntry, std::less<>> words_;
@@ -176,7 +194,7 @@ class Dictionary {
  *
  */
 
-Ref<WordDef> pop_exec_token(vm::Stack& stack);
+Ref<FiftCont> pop_exec_token(vm::Stack& stack);
 Ref<WordList> pop_word_list(vm::Stack& stack);
 void push_argcount(vm::Stack& stack, int args);
 }  // namespace fift

crypto/fift/Fift.cpp

@@ -37,25 +37,18 @@ td::Result<int> Fift::interpret_file(std::string fname, std::string current_dir,
     return td::Status::Error("cannot locate file `" + fname + "`");
   }
   auto file = r_file.move_as_ok();
-  IntCtx ctx;
   std::stringstream ss(file.data);
-  ctx.input_stream = &ss;
-  ctx.filename = td::PathView(file.path).file_name().str();
-  ctx.currentd_dir = td::PathView(file.path).parent_dir().str();
-  ctx.include_depth = is_interactive ? 0 : 1;
-  return do_interpret(ctx);
+  IntCtx ctx{ss, td::PathView(file.path).file_name().str(), td::PathView(file.path).parent_dir().str(),
+             (int)!is_interactive};
+  return do_interpret(ctx, is_interactive);
 }
 
 td::Result<int> Fift::interpret_istream(std::istream& stream, std::string current_dir, bool is_interactive) {
-  IntCtx ctx;
-  ctx.input_stream = &stream;
-  ctx.filename = "stdin";
-  ctx.currentd_dir = current_dir;
-  ctx.include_depth = is_interactive ? 0 : 1;
-  return do_interpret(ctx);
+  IntCtx ctx{stream, "stdin", current_dir, (int)!is_interactive};
+  return do_interpret(ctx, is_interactive);
 }
 
-td::Result<int> Fift::do_interpret(IntCtx& ctx) {
+td::Result<int> Fift::do_interpret(IntCtx& ctx, bool is_interactive) {
   ctx.ton_db = &config_.ton_db;
   ctx.source_lookup = &config_.source_lookup;
   ctx.dictionary = &config_.dictionary;
@@ -64,13 +57,26 @@ td::Result<int> Fift::do_interpret(IntCtx& ctx) {
   if (!ctx.output_stream) {
     return td::Status::Error("Cannot run interpreter without output_stream");
   }
-  try {
-    return funny_interpret_loop(ctx);
-  } catch (fift::IntError ab) {
-    return td::Status::Error(ab.msg);
-  } catch (fift::Quit q) {
-    return q.res;
+  while (true) {
+    auto res = ctx.run(td::make_ref<InterpretCont>());
+    if (res.is_error()) {
+      res = ctx.add_error_loc(res.move_as_error());
+      if (config_.show_backtrace) {
+        std::ostringstream os;
+        ctx.print_error_backtrace(os);
+        LOG(ERROR) << os.str();
+      }
+      if (is_interactive) {
+        LOG(ERROR) << res.move_as_error().message();
+        ctx.top_ctx();
+        ctx.clear_error();
+        ctx.stack.clear();
+        ctx.load_next_line();
+        continue;
+      }
+    }
+    return res;
   }
-  return 0;
 }
+
 }  // namespace fift

crypto/fift/Fift.h

@@ -26,7 +26,6 @@
 
 namespace fift {
 struct IntCtx;
-int funny_interpret_loop(IntCtx& ctx);
 
 struct Fift {
  public:
@@ -36,6 +35,7 @@ struct Fift {
     fift::Dictionary dictionary;
     std::ostream* output_stream{&std::cout};
     std::ostream* error_stream{&std::cerr};
+    bool show_backtrace{true};
   };
   // Fift must own ton_db and dictionary, no concurrent access is allowed
   explicit Fift(Config config);
@@ -48,6 +48,6 @@ struct Fift {
  private:
   Config config_;
 
-  td::Result<int> do_interpret(IntCtx& ctx);
+  td::Result<int> do_interpret(IntCtx& ctx, bool is_interactive = false);
 };
 }  // namespace fift

crypto/fift/IntCtx.cpp

@@ -68,33 +68,69 @@ void CharClassifier::set_char_class(int c, int cl) {
 }
 
 IntCtx::Savepoint::Savepoint(IntCtx& _ctx, std::string new_filename, std::string new_current_dir,
-                             std::istream* new_input_stream)
+                             std::unique_ptr<std::istream> new_input_stream)
     : ctx(_ctx)
     , old_line_no(_ctx.line_no)
     , old_need_line(_ctx.need_line)
     , old_filename(_ctx.filename)
     , old_current_dir(_ctx.currentd_dir)
     , old_input_stream(_ctx.input_stream)
+    , old_input_stream_holder(std::move(_ctx.input_stream_holder))
     , old_curline(_ctx.str)
-    , old_curpos(_ctx.input_ptr - _ctx.str.c_str()) {
+    , old_curpos(_ctx.input_ptr - _ctx.str.c_str())
+    , old_word(_ctx.word) {
   ctx.line_no = 0;
   ctx.filename = new_filename;
   ctx.currentd_dir = new_current_dir;
-  ctx.input_stream = new_input_stream;
+  ctx.input_stream = new_input_stream.get();
+  ctx.input_stream_holder = std::move(new_input_stream);
   ctx.str = "";
   ctx.input_ptr = 0;
   ++(ctx.include_depth);
 }
 
-IntCtx::Savepoint::~Savepoint() {
+bool IntCtx::Savepoint::restore(IntCtx& _ctx) {
+  if (restored || &ctx != &_ctx) {
+    return false;
+  }
   ctx.line_no = old_line_no;
   ctx.need_line = old_need_line;
   ctx.filename = old_filename;
   ctx.currentd_dir = old_current_dir;
   ctx.input_stream = old_input_stream;
+  ctx.input_stream_holder = std::move(old_input_stream_holder);
   ctx.str = old_curline;
   ctx.input_ptr = ctx.str.c_str() + old_curpos;
+  ctx.word = old_word;
   --(ctx.include_depth);
+  return restored = true;
+}
+
+bool IntCtx::enter_ctx(std::string new_filename, std::string new_current_dir,
+                       std::unique_ptr<std::istream> new_input_stream) {
+  if (!new_input_stream) {
+    return false;
+  }
+  ctx_save_stack.emplace_back(*this, std::move(new_filename), std::move(new_current_dir), std::move(new_input_stream));
+  return true;
+}
+
+bool IntCtx::leave_ctx() {
+  if (ctx_save_stack.empty()) {
+    return false;
+  }
+  bool ok = ctx_save_stack.back().restore(*this);
+  ctx_save_stack.pop_back();
+  return ok;
+}
+
+bool IntCtx::top_ctx() {
+  while (!ctx_save_stack.empty()) {
+    if (!leave_ctx()) {
+      return false;
+    }
+  }
+  return true;
 }
 
 bool IntCtx::load_next_line() {
@@ -194,4 +230,99 @@ void IntCtx::check_int_exec() const {
     throw IntError{"internal interpret mode only"};
   }
 }
+
+bool IntCtx::print_error_backtrace(std::ostream& os) const {
+  if (exc_cont.is_null() && exc_next.is_null()) {
+    os << "(no backtrace)\n";
+    return false;
+  }
+  if (exc_cont.not_null()) {
+    os << "top: ";
+    exc_cont->dump(os, *this);
+  }
+  return print_backtrace(os, exc_next);
+}
+
+bool IntCtx::print_backtrace(std::ostream& os, Ref<FiftCont> cont) const {
+  for (int i = 1; cont.not_null() && i <= 16; i++) {
+    os << "level " << i << ": ";
+    cont->dump(os, *this);
+    cont = cont->up();
+  }
+  if (cont.not_null()) {
+    os << "... more levels ...\n";
+  }
+  return true;
+}
+
+Ref<FiftCont> IntCtx::throw_exception(td::Status err, Ref<FiftCont> cur) {
+  exc_cont = std::move(cur);
+  exc_next = std::move(next);
+  error = std::move(err);
+  next.clear();
+  auto cont = std::move(exc_handler);
+  if (cont.is_null()) {
+    return {};  // no Fift exception handler set
+  } else if (cont.is_unique()) {
+    return cont.unique_write().handle_modify(*this);
+  } else {
+    return cont->handle_tail(*this);
+  }
+}
+
+void IntCtx::clear_error() {
+  error = td::Status::OK();
+  exit_code = 0;
+}
+
+td::Result<int> IntCtx::get_result() {
+  if (error.is_error()) {
+    return error.move_as_error();
+  } else {
+    return exit_code;
+  }
+}
+
+td::Status IntCtx::add_error_loc(td::Status err) const {
+  if (err.is_error()) {
+    std::ostringstream os;
+    if (include_depth && line_no) {
+      os << filename << ":" << line_no << ":\t";
+    }
+    if (!word.empty()) {
+      os << word << ":";
+    }
+    return err.move_as_error_prefix(os.str());
+  } else {
+    return err;
+  }
+}
+
+td::Result<int> IntCtx::run(Ref<FiftCont> cont) {
+  clear_error();
+  while (cont.not_null()) {
+    try {
+      if (cont.is_unique()) {
+        cont = cont.unique_write().run_modify(*this);
+      } else {
+        cont = cont->run_tail(*this);
+      }
+    } catch (IntError& err) {
+      cont = throw_exception(td::Status::Error(err.msg), std::move(cont));
+    } catch (vm::VmError& err) {
+      cont = throw_exception(err.as_status(), std::move(cont));
+    } catch (vm::VmVirtError& err) {
+      cont = throw_exception(err.as_status(), std::move(cont));
+    } catch (vm::CellBuilder::CellWriteError&) {
+      cont = throw_exception(td::Status::Error("Cell builder write error"), std::move(cont));
+    } catch (vm::VmFatal&) {
+      cont = throw_exception(td::Status::Error("fatal vm error"), std::move(cont));
+    }
+    if (cont.is_null()) {
+      cont = std::move(next);
+    }
+  }
+  return get_result();
+}
+
 }  // namespace fift

crypto/fift/IntCtx.h

@@ -22,6 +22,11 @@
 #include "crypto/vm/stack.hpp"
 #include "crypto/common/bitstring.h"
 
+#include "td/utils/Status.h"
+
+#include "Dictionary.h"
+#include "Continuation.h"
+
 #include <cstdint>
 #include <cstring>
 #include <iostream>
@@ -65,13 +70,18 @@ class CharClassifier {
 
 struct IntCtx {
   vm::Stack stack;
+  Ref<FiftCont> next, exc_handler;
+  Ref<FiftCont> exc_cont, exc_next;
   int state{0};
   int include_depth{0};
   int line_no{0};
+  int exit_code{0};
+  td::Status error;
   bool need_line{true};
   std::string filename;
   std::string currentd_dir;
   std::istream* input_stream{nullptr};
+  std::unique_ptr<std::istream> input_stream_holder;
   std::ostream* output_stream{nullptr};
   std::ostream* error_stream{nullptr};
 
@@ -79,18 +89,50 @@ struct IntCtx {
   Dictionary* dictionary{nullptr};
   SourceLookup* source_lookup{nullptr};
   int* now{nullptr};
+  std::string word;
 
  private:
   std::string str;
-  const char* input_ptr;
+  const char* input_ptr = nullptr;
+
+  class Savepoint {
+    IntCtx& ctx;
+    int old_line_no;
+    bool old_need_line;
+    bool restored{false};
+    std::string old_filename;
+    std::string old_current_dir;
+    std::istream* old_input_stream;
+    std::unique_ptr<std::istream> old_input_stream_holder;
+    std::string old_curline;
+    std::ptrdiff_t old_curpos;
+    std::string old_word;
+
+   public:
+    Savepoint(IntCtx& _ctx, std::string new_filename, std::string new_current_dir,
+              std::unique_ptr<std::istream> new_input_stream);
+    bool restore(IntCtx& _ctx);
+  };
+
+  std::vector<Savepoint> ctx_save_stack;
 
  public:
   IntCtx() = default;
+  IntCtx(std::istream& _istream, std::string _filename, std::string _curdir = "", int _depth = 0)
+      : include_depth(_depth)
+      , filename(std::move(_filename))
+      , currentd_dir(std::move(_curdir))
+      , input_stream(&_istream) {
+  }
 
   operator vm::Stack&() {
     return stack;
   }
 
+  bool enter_ctx(std::string new_filename, std::string new_current_dir, std::unique_ptr<std::istream> new_input_stream);
+  bool leave_ctx();
+  bool top_ctx();
+
   td::Slice scan_word_to(char delim, bool err_endl = true);
   td::Slice scan_word();
   td::Slice scan_word_ext(const CharClassifier& classifier);
@@ -127,25 +169,29 @@ struct IntCtx {
     state = 0;
     stack.clear();
   }
-  class Savepoint {
-    IntCtx& ctx;
-    int old_line_no;
-    bool old_need_line;
-    std::string old_filename;
-    std::string old_current_dir;
-    std::istream* old_input_stream;
-    std::string old_curline;
-    std::ptrdiff_t old_curpos;
-
-   public:
-    Savepoint(IntCtx& _ctx, std::string new_filename, std::string new_current_dir, std::istream* new_input_stream);
-    ~Savepoint();
-  };
 
   void check_compile() const;
   void check_execute() const;
   void check_not_int_exec() const;
   void check_int_exec() const;
+
+  bool print_error_backtrace(std::ostream& os) const;
+  bool print_backtrace(std::ostream& os, Ref<FiftCont> cont) const;
+
+  td::Status add_error_loc(td::Status err) const;
+
+  void set_exit_code(int err_code) {
+    exit_code = err_code;
+  }
+  int get_exit_code() const {
+    return exit_code;
+  }
+
+  void clear_error();
+  td::Result<int> get_result();
+
+  Ref<FiftCont> throw_exception(td::Status err, Ref<FiftCont> cur = {});
+  td::Result<int> run(Ref<FiftCont> cont);
 };
 
 td::StringBuilder& operator<<(td::StringBuilder& os, const IntCtx& ctx);

crypto/fift/utils.cpp

@@ -28,6 +28,12 @@ namespace {
 std::string fift_dir(std::string dir) {
   return dir.size() > 0 ? dir : td::PathView(td::realpath(__FILE__).move_as_ok()).parent_dir().str() + "lib/";
 }
+std::string smartcont_dir(std::string dir) {
+  return dir.size() > 0
+             ? dir
+             : td::PathView(td::PathView(td::realpath(__FILE__).move_as_ok()).parent_dir_noslash()).parent_dir().str() +
+                   "smartcont/";
+}
 td::Result<std::string> load_source(std::string name, std::string dir = "") {
   return td::read_file_str(fift_dir(dir) + name);
 }
@@ -49,6 +55,9 @@ td::Result<std::string> load_Lisp_fif(std::string dir = "") {
 td::Result<std::string> load_GetOpt_fif(std::string dir = "") {
   return load_source("GetOpt.fif", dir);
 }
+td::Result<std::string> load_wallet3_code_fif(std::string dir = "") {
+  return td::read_file_str(smartcont_dir(dir) + "wallet-v3-code.fif");
+}
 
 class MemoryFileLoader : public fift::FileLoader {
  public:
@@ -98,7 +107,7 @@ class MemoryFileLoader : public fift::FileLoader {
 
 td::Result<fift::SourceLookup> create_source_lookup(std::string main, bool need_preamble = true, bool need_asm = true,
                                                     bool need_ton_util = true, bool need_lisp = true,
-                                                    std::string dir = "") {
+                                                    bool need_w3_code = true, std::string dir = "") {
   auto loader = std::make_unique<MemoryFileLoader>();
   loader->add_file("/main.fif", std::move(main));
   if (need_preamble) {
@@ -127,6 +136,10 @@ td::Result<fift::SourceLookup> create_source_lookup(std::string main, bool need_
     TRY_RESULT(f, load_Lisp_fif(dir));
     loader->add_file("/Lisp.fif", std::move(f));
   }
+  if (need_w3_code) {
+    TRY_RESULT(f, load_wallet3_code_fif(dir));
+    loader->add_file("/wallet-v3-code.fif", std::move(f));
+  }
   auto res = fift::SourceLookup(std::move(loader));
   res.add_include_path("/");
   return std::move(res);
@@ -158,7 +171,7 @@ td::Result<fift::SourceLookup> run_fift(fift::SourceLookup source_lookup, std::o
 }  // namespace
 td::Result<FiftOutput> mem_run_fift(std::string source, std::vector<std::string> args, std::string fift_dir) {
   std::stringstream ss;
-  TRY_RESULT(source_lookup, create_source_lookup(source, true, true, true, true, fift_dir));
+  TRY_RESULT(source_lookup, create_source_lookup(source, true, true, true, true, true, fift_dir));
   TRY_RESULT_ASSIGN(source_lookup, run_fift(std::move(source_lookup), &ss, true, std::move(args)));
   FiftOutput res;
   res.source_lookup = std::move(source_lookup);
@@ -174,8 +187,9 @@ td::Result<FiftOutput> mem_run_fift(SourceLookup source_lookup, std::vector<std:
   return std::move(res);
 }
 td::Result<fift::SourceLookup> create_mem_source_lookup(std::string main, std::string fift_dir, bool need_preamble,
-                                                        bool need_asm, bool need_ton_util, bool need_lisp) {
-  return create_source_lookup(main, need_preamble, need_asm, need_ton_util, need_lisp, fift_dir);
+                                                        bool need_asm, bool need_ton_util, bool need_lisp,
+                                                        bool need_w3_code) {
+  return create_source_lookup(main, need_preamble, need_asm, need_ton_util, need_lisp, need_w3_code, fift_dir);
 }
 
 td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code, std::string fift_dir, bool is_raw) {
@@ -183,7 +197,7 @@ td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code, std::string fift_d
   TRY_RESULT(source_lookup,
              create_source_lookup(PSTRING() << "\"Asm.fif\" include\n " << (is_raw ? "<{" : "") << asm_code << "\n"
                                             << (is_raw ? "}>c" : "") << " boc>B \"res\" B>file",
-                                  true, true, true, false, fift_dir));
+                                  true, true, true, false, false, fift_dir));
   TRY_RESULT(res, run_fift(std::move(source_lookup), &ss));
   TRY_RESULT(boc, res.read_file("res"));
   return vm::std_boc_deserialize(std::move(boc.data));

crypto/fift/utils.h

@@ -28,7 +28,8 @@ struct FiftOutput {
 };
 td::Result<fift::SourceLookup> create_mem_source_lookup(std::string main, std::string fift_dir = "",
                                                         bool need_preamble = true, bool need_asm = true,
-                                                        bool need_ton_util = true, bool need_lisp = true);
+                                                        bool need_ton_util = true, bool need_lisp = true,
+                                                        bool need_w3_code = true);
 td::Result<FiftOutput> mem_run_fift(std::string source, std::vector<std::string> args = {}, std::string fift_dir = "");
 td::Result<FiftOutput> mem_run_fift(SourceLookup source_lookup, std::vector<std::string> args);
 td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code, std::string fift_dir = "", bool is_raw = true);

crypto/fift/words.cpp

@@ -62,7 +62,7 @@ void show_total_cells(std::ostream& stream) {
   stream << "total cells = " << vm::DataCell::get_total_data_cells() << std::endl;
 }
 
-void do_compile(vm::Stack& stack, Ref<WordDef> word_def);
+void do_compile(vm::Stack& stack, Ref<FiftCont> word_def);
 void do_compile_literals(vm::Stack& stack, int count);
 
 void interpret_dot(IntCtx& ctx, bool space_after) {
@@ -204,18 +204,6 @@ void interpret_negate(vm::Stack& stack) {
   stack.push_int(-stack.pop_int());
 }
 
-void interpret_const(vm::Stack& stack, long long val) {
-  stack.push_smallint(val);
-}
-
-void interpret_big_const(vm::Stack& stack, td::RefInt256 val) {
-  stack.push_int(std::move(val));
-}
-
-void interpret_literal(vm::Stack& stack, vm::StackEntry se) {
-  stack.push(std::move(se));
-}
-
 void interpret_cmp(vm::Stack& stack, const char opt[3]) {
   auto y = stack.pop_int();
   auto x = stack.pop_int();
@@ -537,7 +525,8 @@ void interpret_hold(vm::Stack& stack) {
   stack.check_underflow(2);
   char buffer[8];
   unsigned len = make_utf8_char(buffer, stack.pop_smallint_range(0x10ffff, -128));
-  std::string s = stack.pop_string() + std::string{buffer, len};
+  std::string s = stack.pop_string();
+  s.append(buffer, len);
   stack.push_string(std::move(s));
 }
 
@@ -1567,78 +1556,183 @@ void interpret_dict_get(vm::Stack& stack, int sgnd, int mode) {
   }
 }
 
-void interpret_dict_map(IntCtx& ctx, bool ext, bool sgnd) {
-  auto func = pop_exec_token(ctx);
-  int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits);
-  vm::Dictionary dict{ctx.stack.pop_maybe_cell(), n}, dict2{n};
-  for (auto entry : dict.range(false, sgnd)) {
-    ctx.stack.push_builder(Ref<vm::CellBuilder>{true});
-    if (ext) {
-      ctx.stack.push_int(dict.key_as_integer(entry.first, sgnd));
-    }
-    ctx.stack.push_cellslice(std::move(entry.second));
-    func->run(ctx);
-    if (ctx.stack.pop_bool()) {
-      if (!dict2.set_builder(entry.first, n, ctx.stack.pop_builder())) {
-        throw IntError{"cannot insert value into dictionary"};
-      }
-    }
-  };
-  ctx.stack.push_maybe_cell(std::move(dict2).extract_root_cell());
+class DictMapCont : public LoopCont {
+  int n;
+  bool ext;
+  bool sgnd;
+  vm::Dictionary dict, dict2;
+  vm::DictIterator it;
+
+ public:
+  DictMapCont(Ref<FiftCont> _func, Ref<FiftCont> _after, int _n, Ref<vm::Cell> dict_root, bool _ext, bool _sgnd)
+      : LoopCont(std::move(_func), std::move(_after))
+      , n(_n)
+      , ext(_ext)
+      , sgnd(_sgnd)
+      , dict(std::move(dict_root), n)
+      , dict2(n) {
+  }
+  DictMapCont(const DictMapCont&) = default;
+  DictMapCont* make_copy() const override {
+    return new DictMapCont(*this);
+  }
+  bool init(IntCtx& ctx) override {
+    it = dict.init_iterator(false, sgnd);
+    return true;
+  }
+  bool pre_exec(IntCtx& ctx) override;
+  bool post_exec(IntCtx& ctx) override;
+  bool finalize(IntCtx& ctx) override;
+};
+
+bool DictMapCont::pre_exec(IntCtx& ctx) {
+  if (it.eof()) {
+    return false;
+  }
+  ctx.stack.push_builder(td::make_ref<vm::CellBuilder>());
+  if (ext) {
+    ctx.stack.push_int(dict.key_as_integer(it.cur_pos(), sgnd));
+  }
+  ctx.stack.push_cellslice(it.cur_value());
+  return true;
 }
 
-void interpret_dict_foreach(IntCtx& ctx, bool reverse, bool sgnd) {
+bool DictMapCont::post_exec(IntCtx& ctx) {
+  if (ctx.stack.pop_bool()) {
+    if (!dict2.set_builder(it.cur_pos(), n, ctx.stack.pop_builder())) {
+      throw IntError{"cannot insert value into dictionary"};
+    }
+  }
+  return !(++it).eof();
+}
+
+bool DictMapCont::finalize(IntCtx& ctx) {
+  ctx.stack.push_maybe_cell(std::move(dict2).extract_root_cell());
+  return true;
+}
+
+Ref<FiftCont> interpret_dict_map(IntCtx& ctx, bool ext, bool sgnd) {
   auto func = pop_exec_token(ctx);
   int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits);
-  vm::Dictionary dict{ctx.stack.pop_maybe_cell(), n};
-  for (auto entry : dict.range(reverse, sgnd)) {
-    ctx.stack.push_int(dict.key_as_integer(entry.first, sgnd));
-    ctx.stack.push_cellslice(std::move(entry.second));
-    func->run(ctx);
-    if (!ctx.stack.pop_bool()) {
-      ctx.stack.push_bool(false);
-      return;
-    }
-  };
-  ctx.stack.push_bool(true);
+  return td::make_ref<DictMapCont>(std::move(func), std::move(ctx.next), n, ctx.stack.pop_maybe_cell(), ext, sgnd);
+}
+
+class DictIterCont : public LoopCont {
+  int n;
+  bool reverse;
+  bool sgnd;
+  bool ok;
+  bool inited{false};
+  vm::Dictionary dict;
+  vm::DictIterator it;
+
+ public:
+  DictIterCont(Ref<FiftCont> _func, Ref<FiftCont> _after, int _n, Ref<vm::Cell> dict_root, bool _reverse, bool _sgnd)
+      : LoopCont(std::move(_func), std::move(_after))
+      , n(_n)
+      , reverse(_reverse)
+      , sgnd(_sgnd)
+      , ok(true)
+      , dict(std::move(dict_root), n) {
+  }
+  DictIterCont(const DictIterCont&) = default;
+  DictIterCont* make_copy() const override {
+    return new DictIterCont(*this);
+  }
+  bool do_init();
+  bool init(IntCtx& ctx) override {
+    return do_init();
+  }
+  bool pre_exec(IntCtx& ctx) override;
+  bool post_exec(IntCtx& ctx) override;
+  bool finalize(IntCtx& ctx) override;
+  template <typename T>
+  bool lookup(const T& key, bool strict, bool backw) {
+    return do_init() && it.lookup(key, strict, backw);
+  }
+};
+
+bool DictIterCont::do_init() {
+  if (!inited) {
+    it = dict.init_iterator(reverse, sgnd);
+    inited = true;
+  }
+  return true;
+}
+
+bool DictIterCont::pre_exec(IntCtx& ctx) {
+  if (it.eof()) {
+    return false;
+  }
+  ctx.stack.push_int(dict.key_as_integer(it.cur_pos(), sgnd));
+  ctx.stack.push_cellslice(it.cur_value());
+  return true;
+}
+
+bool DictIterCont::post_exec(IntCtx& ctx) {
+  ok = ctx.stack.pop_bool();
+  return ok && !(++it).eof();
+}
+
+bool DictIterCont::finalize(IntCtx& ctx) {
+  ctx.stack.push_bool(ok);
+  return true;
+}
+
+Ref<FiftCont> interpret_dict_foreach(IntCtx& ctx, bool reverse, bool sgnd) {
+  auto func = pop_exec_token(ctx);
+  int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits);
+  return td::make_ref<DictIterCont>(std::move(func), std::move(ctx.next), n, ctx.stack.pop_maybe_cell(), reverse, sgnd);
 }
 
 // mode: +1 = reverse, +2 = signed, +4 = strict, +8 = lookup backwards, +16 = with hint
-void interpret_dict_foreach_from(IntCtx& ctx, int mode) {
+Ref<FiftCont> interpret_dict_foreach_from(IntCtx& ctx, int mode) {
   if (mode < 0) {
     mode = ctx.stack.pop_smallint_range(31);
   }
   auto func = pop_exec_token(ctx);
   int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits);
-  vm::Dictionary dict{ctx.stack.pop_maybe_cell(), n};
-  vm::DictIterator it{dict, mode & 3};
-  unsigned char buffer[vm::Dictionary::max_key_bytes];
+  auto it_cont = td::make_ref<DictIterCont>(std::move(func), std::move(ctx.next), n, ctx.stack.pop_maybe_cell(),
+                                            mode & 1, mode & 2);
   for (int s = (mode >> 4) & 1; s >= 0; --s) {
-    auto key = dict.integer_key(ctx.stack.pop_int(), n, mode & 2, buffer);
+    unsigned char buffer[vm::Dictionary::max_key_bytes];
+    auto key = vm::Dictionary::integer_key(ctx.stack.pop_int(), n, mode & 2, buffer);
     if (!key.is_valid()) {
       throw IntError{"not enough bits for a dictionary key"};
     }
-    it.lookup(key, mode & 4, mode & 8);
+    it_cont.write().lookup(key, mode & 4, mode & 8);
   }
-  for (; !it.eof(); ++it) {
-    ctx.stack.push_int(dict.key_as_integer(it.cur_pos(), mode & 2));
-    ctx.stack.push_cellslice(it.cur_value());
-    func->run(ctx);
-    if (!ctx.stack.pop_bool()) {
-      ctx.stack.push_bool(false);
-      return;
-    }
-  };
-  ctx.stack.push_bool(true);
+  return it_cont;
 }
 
-void interpret_dict_merge(IntCtx& ctx) {
-  auto func = pop_exec_token(ctx);
-  int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits);
-  vm::Dictionary dict2{ctx.stack.pop_maybe_cell(), n};
-  vm::Dictionary dict1{ctx.stack.pop_maybe_cell(), n};
-  vm::Dictionary dict3{n};
-  auto it1 = dict1.begin(), it2 = dict2.begin();
+class DictMergeCont : public LoopCont {
+  int n;
+  vm::Dictionary dict1, dict2, dict3;
+  vm::DictIterator it1, it2;
+
+ public:
+  DictMergeCont(Ref<FiftCont> _func, Ref<FiftCont> _after, int _n, Ref<vm::Cell> dict1_root, Ref<vm::Cell> dict2_root)
+      : LoopCont(std::move(_func), std::move(_after))
+      , n(_n)
+      , dict1(std::move(dict1_root), n)
+      , dict2(std::move(dict2_root), n)
+      , dict3(n) {
+  }
+  DictMergeCont(const DictMergeCont&) = default;
+  DictMergeCont* make_copy() const override {
+    return new DictMergeCont(*this);
+  }
+  bool init(IntCtx& ctx) override {
+    it1 = dict1.begin();
+    it2 = dict2.begin();
+    return true;
+  }
+  bool pre_exec(IntCtx& ctx) override;
+  bool post_exec(IntCtx& ctx) override;
+  bool finalize(IntCtx& ctx) override;
+};
+
+bool DictMergeCont::pre_exec(IntCtx& ctx) {
   while (!it1.eof() || !it2.eof()) {
     int c = it1.eof() ? 1 : (it2.eof() ? -1 : it1.cur_pos().compare(it2.cur_pos(), n));
     bool ok = true;
@@ -1652,29 +1746,67 @@ void interpret_dict_merge(IntCtx& ctx) {
       ctx.stack.push_builder(Ref<vm::CellBuilder>{true});
       ctx.stack.push_cellslice(it1.cur_value());
       ctx.stack.push_cellslice(it2.cur_value());
-      func->run(ctx);
-      if (ctx.stack.pop_bool()) {
-        ok = dict3.set_builder(it1.cur_pos(), n, ctx.stack.pop_builder());
-      }
-      ++it1;
-      ++it2;
+      return true;
     }
     if (!ok) {
       throw IntError{"cannot insert value into dictionary"};
     }
   }
-  ctx.stack.push_maybe_cell(std::move(dict3).extract_root_cell());
+  return false;
 }
 
-void interpret_dict_diff(IntCtx& ctx) {
+bool DictMergeCont::post_exec(IntCtx& ctx) {
+  if (ctx.stack.pop_bool() && !dict3.set_builder(it1.cur_pos(), n, ctx.stack.pop_builder())) {
+    throw IntError{"cannot insert value into dictionary"};
+  }
+  ++it1;
+  ++it2;
+  return true;
+}
+
+bool DictMergeCont::finalize(IntCtx& ctx) {
+  ctx.stack.push_maybe_cell(std::move(dict3).extract_root_cell());
+  return true;
+}
+
+Ref<FiftCont> interpret_dict_merge(IntCtx& ctx) {
   auto func = pop_exec_token(ctx);
   int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits);
-  vm::Dictionary dict2{ctx.stack.pop_maybe_cell(), n};
-  vm::Dictionary dict1{ctx.stack.pop_maybe_cell(), n};
-  auto it1 = dict1.begin(), it2 = dict2.begin();
+  auto dict2_root = ctx.stack.pop_maybe_cell();
+  return td::make_ref<DictMergeCont>(std::move(func), std::move(ctx.next), n, ctx.stack.pop_maybe_cell(),
+                                     std::move(dict2_root));
+}
+
+class DictDiffCont : public LoopCont {
+  int n;
+  bool ok{true};
+  vm::Dictionary dict1, dict2;
+  vm::DictIterator it1, it2;
+
+ public:
+  DictDiffCont(Ref<FiftCont> _func, Ref<FiftCont> _after, int _n, Ref<vm::Cell> dict1_root, Ref<vm::Cell> dict2_root)
+      : LoopCont(std::move(_func), std::move(_after))
+      , n(_n)
+      , dict1(std::move(dict1_root), n)
+      , dict2(std::move(dict2_root), n) {
+  }
+  DictDiffCont(const DictDiffCont&) = default;
+  DictDiffCont* make_copy() const override {
+    return new DictDiffCont(*this);
+  }
+  bool init(IntCtx& ctx) override {
+    it1 = dict1.begin();
+    it2 = dict2.begin();
+    return true;
+  }
+  bool pre_exec(IntCtx& ctx) override;
+  bool post_exec(IntCtx& ctx) override;
+  bool finalize(IntCtx& ctx) override;
+};
+
+bool DictDiffCont::pre_exec(IntCtx& ctx) {
   while (!it1.eof() || !it2.eof()) {
     int c = it1.eof() ? 1 : (it2.eof() ? -1 : it1.cur_pos().compare(it2.cur_pos(), n));
-    bool run = true;
     if (c < 0) {
       ctx.stack.push_int(dict1.key_as_integer(it1.cur_pos()));
       ctx.stack.push_cellslice(it1.cur_value());
@@ -1691,20 +1823,33 @@ void interpret_dict_diff(IntCtx& ctx) {
         ctx.stack.push_cellslice(it1.cur_value());
         ctx.stack.push_cellslice(it2.cur_value());
       } else {
-        run = false;
+        ++it1;
+        ++it2;
+        continue;
       }
       ++it1;
       ++it2;
     }
-    if (run) {
-      func->run(ctx);
-      if (!ctx.stack.pop_bool()) {
-        ctx.stack.push_bool(false);
-        return;
-      }
-    }
+    return true;
   }
-  ctx.stack.push_bool(true);
+  return false;
+}
+
+bool DictDiffCont::post_exec(IntCtx& ctx) {
+  return (ok = ctx.stack.pop_bool());
+}
+
+bool DictDiffCont::finalize(IntCtx& ctx) {
+  ctx.stack.push_bool(ok);
+  return true;
+}
+
+Ref<FiftCont> interpret_dict_diff(IntCtx& ctx) {
+  auto func = pop_exec_token(ctx);
+  int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits);
+  auto dict2_root = ctx.stack.pop_maybe_cell();
+  return td::make_ref<DictDiffCont>(std::move(func), std::move(ctx.next), n, ctx.stack.pop_maybe_cell(),
+                                    std::move(dict2_root));
 }
 
 void interpret_pfx_dict_add(vm::Stack& stack, vm::Dictionary::SetMode mode, bool add_builder) {
@@ -1791,7 +1936,7 @@ void interpret_wordlist_begin(IntCtx& ctx) {
 void interpret_wordlist_end_aux(vm::Stack& stack) {
   Ref<WordList> wordlist_ref = pop_word_list(stack);
   wordlist_ref.write().close();
-  stack.push({vm::from_object, Ref<WordDef>{wordlist_ref}});
+  stack.push({vm::from_object, Ref<FiftCont>{wordlist_ref}});
 }
 
 void interpret_wordlist_end(IntCtx& ctx) {
@@ -1846,7 +1991,7 @@ void interpret_create(IntCtx& ctx) {
   interpret_create_aux(ctx, 0);
 }
 
-Ref<WordDef> create_aux_wd{Ref<CtxWord>{true, std::bind(interpret_create_aux, std::placeholders::_1, -1)}};
+Ref<FiftCont> create_aux_wd{Ref<CtxWord>{true, std::bind(interpret_create_aux, std::placeholders::_1, -1)}};
 
 // { bl word <mode> 2 ' (create) } :: :
 void interpret_colon(IntCtx& ctx, int mode) {
@@ -2027,39 +2172,43 @@ void interpret_parse_hex_number(vm::Stack& stack) {
 }
 
 void interpret_quit(IntCtx& ctx) {
-  throw Quit{0};
+  // TODO: change to correct behavior
+  ctx.set_exit_code(0);
+  ctx.next.clear();
 }
 
 void interpret_bye(IntCtx& ctx) {
-  throw Quit{-1};
+  ctx.set_exit_code(-1);
+  ctx.next.clear();
 }
 
-void interpret_halt(vm::Stack& stack) {
-  int code = stack.pop_smallint_range(255);
-  throw Quit{~code};
+void interpret_halt(IntCtx& ctx) {
+  ctx.set_exit_code(~ctx.stack.pop_smallint_range(255));
+  ctx.next.clear();
 }
 
 void interpret_abort(IntCtx& ctx) {
   throw IntError{ctx.stack.pop_string()};
 }
 
-Ref<WordDef> interpret_execute(IntCtx& ctx) {
+Ref<FiftCont> interpret_execute(IntCtx& ctx) {
   return pop_exec_token(ctx);
 }
 
-Ref<WordDef> interpret_execute_times(IntCtx& ctx) {
+Ref<FiftCont> interpret_execute_times(IntCtx& ctx) {
   int count = ctx.stack.pop_smallint_range(1000000000);
-  auto wd_ref = pop_exec_token(ctx);
-  if (!count) {
+  auto body = pop_exec_token(ctx);
+  if (count <= 0) {
     return {};
   }
-  while (--count > 0) {
-    wd_ref->run(ctx);
+  if (count == 1) {
+    return body;
   }
-  return wd_ref;
+  ctx.next = td::make_ref<TimesCont>(body, std::move(ctx.next), count - 1);
+  return body;
 }
 
-Ref<WordDef> interpret_if(IntCtx& ctx) {
+Ref<FiftCont> interpret_if(IntCtx& ctx) {
   auto true_ref = pop_exec_token(ctx);
   if (ctx.stack.pop_bool()) {
     return true_ref;
@@ -2068,7 +2217,7 @@ Ref<WordDef> interpret_if(IntCtx& ctx) {
   }
 }
 
-Ref<WordDef> interpret_ifnot(IntCtx& ctx) {
+Ref<FiftCont> interpret_ifnot(IntCtx& ctx) {
   auto false_ref = pop_exec_token(ctx);
   if (ctx.stack.pop_bool()) {
     return {};
@@ -2077,7 +2226,7 @@ Ref<WordDef> interpret_ifnot(IntCtx& ctx) {
   }
 }
 
-Ref<WordDef> interpret_cond(IntCtx& ctx) {
+Ref<FiftCont> interpret_cond(IntCtx& ctx) {
   auto false_ref = pop_exec_token(ctx);
   auto true_ref = pop_exec_token(ctx);
   if (ctx.stack.pop_bool()) {
@@ -2087,23 +2236,17 @@ Ref<WordDef> interpret_cond(IntCtx& ctx) {
   }
 }
 
-void interpret_while(IntCtx& ctx) {
-  auto body_ref = pop_exec_token(ctx);
-  auto cond_ref = pop_exec_token(ctx);
-  while (true) {
-    cond_ref->run(ctx);
-    if (!ctx.stack.pop_bool()) {
-      break;
-    }
-    body_ref->run(ctx);
-  }
+Ref<FiftCont> interpret_while(IntCtx& ctx) {
+  auto body = pop_exec_token(ctx);
+  auto cond = pop_exec_token(ctx);
+  ctx.next = td::make_ref<WhileCont>(cond, std::move(body), std::move(ctx.next), true);
+  return cond;
 }
 
-void interpret_until(IntCtx& ctx) {
-  auto body_ref = pop_exec_token(ctx);
-  do {
-    body_ref->run(ctx);
-  } while (!ctx.stack.pop_bool());
+Ref<FiftCont> interpret_until(IntCtx& ctx) {
+  auto body = pop_exec_token(ctx);
+  ctx.next = td::make_ref<UntilCont>(body, std::move(ctx.next));
+  return body;
 }
 
 void interpret_tick(IntCtx& ctx) {
@@ -2133,25 +2276,39 @@ void interpret_find(IntCtx& ctx) {
   }
 }
 
-void interpret_tick_nop(vm::Stack& stack) {
-  stack.push({vm::from_object, Dictionary::nop_word_def});
+void interpret_leave_source(IntCtx& ctx) {
+  if (!ctx.leave_ctx()) {
+    throw IntError{"cannot leave included file interpretation context"};
+  }
 }
 
-void interpret_include(IntCtx& ctx) {
+Ref<FiftCont> interpret_include(IntCtx& ctx) {
   auto fname = ctx.stack.pop_string();
   auto r_file = ctx.source_lookup->lookup_source(fname, ctx.currentd_dir);
   if (r_file.is_error()) {
     throw IntError{"cannot locate file `" + fname + "`"};
   }
   auto file = r_file.move_as_ok();
-  std::stringstream ss(std::move(file.data));
-  IntCtx::Savepoint save{ctx, td::PathView(file.path).file_name().str(), td::PathView(file.path).parent_dir().str(),
-                         &ss};
-  funny_interpret_loop(ctx);
+  auto ss = std::make_unique<std::stringstream>(std::move(file.data));
+  if (!ctx.enter_ctx(td::PathView(file.path).file_name().str(), td::PathView(file.path).parent_dir().str(),
+                     std::move(ss))) {
+    throw IntError{"cannot enter included file interpretation context"};
+  }
+  ctx.next = SeqCont::seq(td::make_ref<CtxWord>(interpret_leave_source), std::move(ctx.next));
+  return td::make_ref<InterpretCont>();
 }
 
-void interpret_skip_source(vm::Stack& stack) {
-  throw SkipToEof();
+td::Ref<vm::Box> exit_interpret{true};
+
+Ref<FiftCont> interpret_skip_source(IntCtx& ctx) {
+  auto cont = exit_interpret->get().as_object<FiftCont>();
+  ctx.next.clear();
+  /*
+  if (cont.is_null()) {
+    throw IntError{"no interpreter exit point set"};
+  }
+  */
+  return cont;
 }
 
 void interpret_words(IntCtx& ctx) {
@@ -2161,6 +2318,14 @@ void interpret_words(IntCtx& ctx) {
   *ctx.output_stream << std::endl;
 }
 
+void interpret_print_backtrace(IntCtx& ctx) {
+  ctx.print_backtrace(*ctx.output_stream, ctx.next);
+}
+
+void interpret_print_continuation(IntCtx& ctx) {
+  ctx.print_backtrace(*ctx.output_stream, pop_exec_token(ctx));
+}
+
 void interpret_pack_std_smc_addr(vm::Stack& stack) {
   block::StdAddress a;
   stack.check_underflow(3);
@@ -2450,17 +2615,17 @@ void interpret_get_fixed_cmdline_arg(vm::Stack& stack, int n) {
 }
 
 // n -- executes $n
-void interpret_get_cmdline_arg(IntCtx& ctx) {
+Ref<FiftCont> interpret_get_cmdline_arg(IntCtx& ctx) {
   int n = ctx.stack.pop_smallint_range(999999);
   if (n) {
     interpret_get_fixed_cmdline_arg(ctx.stack, n);
-    return;
+    return {};
   }
   auto entry = ctx.dictionary->lookup("$0 ");
   if (!entry) {
     throw IntError{"-?"};
   } else {
-    (*entry)(ctx);
+    return entry->get_def();
   }
 }
 
@@ -2493,16 +2658,16 @@ void interpret_getenv_exists(vm::Stack& stack) {
 }
 
 // x1 .. xn n 'w -->
-void interpret_execute_internal(IntCtx& ctx) {
-  Ref<WordDef> word_def = pop_exec_token(ctx);
+Ref<FiftCont> interpret_execute_internal(IntCtx& ctx) {
+  Ref<FiftCont> word_def = pop_exec_token(ctx);
   int count = ctx.stack.pop_smallint_range(255);
   ctx.stack.check_underflow(count);
-  word_def->run(ctx);
+  return word_def;
 }
 
 // wl x1 .. xn n 'w --> wl'
 void interpret_compile_internal(vm::Stack& stack) {
-  Ref<WordDef> word_def = pop_exec_token(stack);
+  Ref<FiftCont> word_def = pop_exec_token(stack);
   int count = stack.pop_smallint_range(255);
   do_compile_literals(stack, count);
   if (word_def != Dictionary::nop_word_def) {
@@ -2510,12 +2675,14 @@ void interpret_compile_internal(vm::Stack& stack) {
   }
 }
 
-void do_compile(vm::Stack& stack, Ref<WordDef> word_def) {
+void do_compile(vm::Stack& stack, Ref<FiftCont> word_def) {
   Ref<WordList> wl_ref = pop_word_list(stack);
   if (word_def != Dictionary::nop_word_def) {
-    if ((td::uint64)word_def->list_size() <= 1) {
+    auto list_size = word_def->list_size();
+    if ((td::uint64)list_size <= 1) {
       // inline short definitions
-      wl_ref.write().append(*(word_def->get_list()));
+      auto list = word_def->get_list();
+      wl_ref.write().append(list, list + list_size);
     } else {
       wl_ref.write().push_back(word_def);
     }
@@ -2524,19 +2691,7 @@ void do_compile(vm::Stack& stack, Ref<WordDef> word_def) {
 }
 
 void compile_one_literal(WordList& wlist, vm::StackEntry val) {
-  using namespace std::placeholders;
-  if (val.type() == vm::StackEntry::t_int) {
-    auto x = std::move(val).as_int();
-    if (!x->signed_fits_bits(257)) {
-      throw IntError{"invalid numeric literal"};
-    } else if (x->signed_fits_bits(td::BigIntInfo::word_shift)) {
-      wlist.push_back(Ref<StackWord>{true, std::bind(interpret_const, _1, x->to_long())});
-    } else {
-      wlist.push_back(Ref<StackWord>{true, std::bind(interpret_big_const, _1, std::move(x))});
-    }
-  } else {
-    wlist.push_back(Ref<StackWord>{true, std::bind(interpret_literal, _1, std::move(val))});
-  }
+  wlist.push_back(LitCont::literal(std::move(val)));
 }
 
 void do_compile_literals(vm::Stack& stack, int count) {
@@ -2548,8 +2703,16 @@ void do_compile_literals(vm::Stack& stack, int count) {
   if (wl_ref.is_null()) {
     throw IntError{"list of words expected"};
   }
-  for (int i = count - 1; i >= 0; i--) {
-    compile_one_literal(wl_ref.write(), std::move(stack[i]));
+  if (count >= 2) {
+    std::vector<vm::StackEntry> literals;
+    for (int i = count - 1; i >= 0; i--) {
+      literals.push_back(std::move(stack[i]));
+    }
+    wl_ref.write().push_back(td::make_ref<MultiLitCont>(std::move(literals)));
+  } else {
+    for (int i = count - 1; i >= 0; i--) {
+      compile_one_literal(wl_ref.write(), std::move(stack[i]));
+    }
   }
   stack.pop_many(count + 1);
   stack.push({vm::from_object, wl_ref});
@@ -2654,13 +2817,13 @@ void init_words_common(Dictionary& d) {
   d.def_stack_word("or ", interpret_or);
   d.def_stack_word("xor ", interpret_xor);
   // integer constants
-  d.def_stack_word("false ", std::bind(interpret_const, _1, 0));
-  d.def_stack_word("true ", std::bind(interpret_const, _1, -1));
-  d.def_stack_word("0 ", std::bind(interpret_const, _1, 0));
-  d.def_stack_word("1 ", std::bind(interpret_const, _1, 1));
-  d.def_stack_word("2 ", std::bind(interpret_const, _1, 2));
-  d.def_stack_word("-1 ", std::bind(interpret_const, _1, -1));
-  d.def_stack_word("bl ", std::bind(interpret_const, _1, 32));
+  d.def_word("false ", IntLitCont::literal(0));
+  d.def_word("true ", IntLitCont::literal(-1));
+  d.def_word("0 ", IntLitCont::literal(0));
+  d.def_word("1 ", IntLitCont::literal(1));
+  d.def_word("2 ", IntLitCont::literal(2));
+  d.def_word("-1 ", IntLitCont::literal(-1));
+  d.def_word("bl ", IntLitCont::literal(32));
   // integer comparison
   d.def_stack_word("cmp ", std::bind(interpret_cmp, _1, "\xff\x00\x01"));
   d.def_stack_word("= ", std::bind(interpret_cmp, _1, "\x00\xff\x00"));
@@ -2835,16 +2998,16 @@ void init_words_common(Dictionary& d) {
   d.def_stack_word("pfxdict!+ ", std::bind(interpret_pfx_dict_add, _1, vm::Dictionary::SetMode::Add, false));
   d.def_stack_word("pfxdict! ", std::bind(interpret_pfx_dict_add, _1, vm::Dictionary::SetMode::Set, false));
   d.def_stack_word("pfxdict@ ", interpret_pfx_dict_get);
-  d.def_ctx_word("dictmap ", std::bind(interpret_dict_map, _1, false, false));
-  d.def_ctx_word("dictmapext ", std::bind(interpret_dict_map, _1, true, false));
-  d.def_ctx_word("idictmapext ", std::bind(interpret_dict_map, _1, true, true));
-  d.def_ctx_word("dictforeach ", std::bind(interpret_dict_foreach, _1, false, false));
-  d.def_ctx_word("idictforeach ", std::bind(interpret_dict_foreach, _1, false, true));
-  d.def_ctx_word("dictforeachrev ", std::bind(interpret_dict_foreach, _1, true, false));
-  d.def_ctx_word("idictforeachrev ", std::bind(interpret_dict_foreach, _1, true, true));
-  d.def_ctx_word("dictforeachfromx ", std::bind(interpret_dict_foreach_from, _1, -1));
-  d.def_ctx_word("dictmerge ", interpret_dict_merge);
-  d.def_ctx_word("dictdiff ", interpret_dict_diff);
+  d.def_ctx_tail_word("dictmap ", std::bind(interpret_dict_map, _1, false, false));
+  d.def_ctx_tail_word("dictmapext ", std::bind(interpret_dict_map, _1, true, false));
+  d.def_ctx_tail_word("idictmapext ", std::bind(interpret_dict_map, _1, true, true));
+  d.def_ctx_tail_word("dictforeach ", std::bind(interpret_dict_foreach, _1, false, false));
+  d.def_ctx_tail_word("idictforeach ", std::bind(interpret_dict_foreach, _1, false, true));
+  d.def_ctx_tail_word("dictforeachrev ", std::bind(interpret_dict_foreach, _1, true, false));
+  d.def_ctx_tail_word("idictforeachrev ", std::bind(interpret_dict_foreach, _1, true, true));
+  d.def_ctx_tail_word("dictforeachfromx ", std::bind(interpret_dict_foreach_from, _1, -1));
+  d.def_ctx_tail_word("dictmerge ", interpret_dict_merge);
+  d.def_ctx_tail_word("dictdiff ", interpret_dict_diff);
   // slice/bitstring constants
   d.def_active_word("x{", interpret_bitstring_hex_literal);
   d.def_active_word("b{", interpret_bitstring_binary_literal);
@@ -2880,8 +3043,8 @@ void init_words_common(Dictionary& d) {
   d.def_ctx_tail_word("if ", interpret_if);
   d.def_ctx_tail_word("ifnot ", interpret_ifnot);
   d.def_ctx_tail_word("cond ", interpret_cond);
-  d.def_ctx_word("while ", interpret_while);
-  d.def_ctx_word("until ", interpret_until);
+  d.def_ctx_tail_word("while ", interpret_while);
+  d.def_ctx_tail_word("until ", interpret_until);
   // compiler control
   d.def_active_word("[ ", interpret_internal_interpret_begin);
   d.def_active_word("] ", interpret_internal_interpret_end);
@@ -2890,9 +3053,9 @@ void init_words_common(Dictionary& d) {
   d.def_stack_word("({) ", interpret_wordlist_begin_aux);
   d.def_stack_word("(}) ", interpret_wordlist_end_aux);
   d.def_stack_word("(compile) ", interpret_compile_internal);
-  d.def_ctx_word("(execute) ", interpret_execute_internal);
+  d.def_ctx_tail_word("(execute) ", interpret_execute_internal);
   d.def_active_word("' ", interpret_tick);
-  d.def_stack_word("'nop ", interpret_tick_nop);
+  d.def_word("'nop ", LitCont::literal({vm::from_object, Dictionary::nop_word_def}));
   // dictionary manipulation
   d.def_ctx_word("find ", interpret_find);
   d.def_ctx_word("create ", interpret_create);
@@ -2904,20 +3067,23 @@ void init_words_common(Dictionary& d) {
   d.def_ctx_word("(forget) ", interpret_forget_aux);
   d.def_ctx_word("forget ", interpret_forget);
   d.def_ctx_word("words ", interpret_words);
+  d.def_ctx_word(".bt ", interpret_print_backtrace);
+  d.def_ctx_word("cont. ", interpret_print_continuation);
   // input parse
   d.def_ctx_word("word ", interpret_word);
   d.def_ctx_word("(word) ", interpret_word_ext);
   d.def_ctx_word("skipspc ", interpret_skipspc);
-  d.def_ctx_word("include ", interpret_include);
-  d.def_stack_word("skip-to-eof ", interpret_skip_source);
+  d.def_ctx_tail_word("include ", interpret_include);
+  d.def_ctx_tail_word("skip-to-eof ", interpret_skip_source);
+  d.def_word("'exit-interpret ", LitCont::literal(exit_interpret));
   d.def_ctx_word("abort ", interpret_abort);
   d.def_ctx_word("quit ", interpret_quit);
   d.def_ctx_word("bye ", interpret_bye);
-  d.def_stack_word("halt ", interpret_halt);
+  d.def_ctx_word("halt ", interpret_halt);
   // cmdline args
-  d.def_stack_word("$* ", std::bind(interpret_literal, _1, vm::StackEntry{cmdline_args}));
+  d.def_word("$* ", LitCont::literal(cmdline_args));
   d.def_stack_word("$# ", interpret_get_cmdline_arg_count);
-  d.def_ctx_word("$() ", interpret_get_cmdline_arg);
+  d.def_ctx_tail_word("$() ", interpret_get_cmdline_arg);
 }
 
 void init_words_ton(Dictionary& d) {
@@ -2934,7 +3100,7 @@ void init_words_vm(Dictionary& d, bool enable_debug) {
   using namespace std::placeholders;
   vm::init_op_cp0(enable_debug);
   // vm run
-  d.def_stack_word("vmlibs ", std::bind(interpret_literal, _1, vm::StackEntry{vm_libraries}));
+  d.def_word("vmlibs ", LitCont::literal(vm_libraries));
   // d.def_ctx_word("runvmcode ", std::bind(interpret_run_vm, _1, 0x40));
   // d.def_ctx_word("runvm ", std::bind(interpret_run_vm, _1, 0x45));
   d.def_ctx_word("runvmx ", std::bind(interpret_run_vm, _1, -1));
@@ -2947,7 +3113,7 @@ void init_words_vm(Dictionary& d, bool enable_debug) {
 void import_cmdline_args(Dictionary& d, std::string arg0, int n, const char* const argv[]) {
   using namespace std::placeholders;
   LOG(DEBUG) << "import_cmdlist_args(" << arg0 << "," << n << ")";
-  d.def_stack_word("$0 ", std::bind(interpret_literal, _1, vm::StackEntry{arg0}));
+  d.def_word("$0 ", LitCont::literal(arg0));
   vm::StackEntry list;
   for (int i = n - 1; i >= 0; i--) {
     list = vm::StackEntry::cons(vm::StackEntry{argv[i]}, list);
@@ -2978,104 +3144,81 @@ td::RefInt256 numeric_value(std::string s) {
   return num;
 }
 
-int funny_interpret_loop(IntCtx& ctx) {
-  while (ctx.load_next_line()) {
-    if (ctx.is_sb()) {
-      continue;
-    }
-    std::ostringstream errs;
-    bool ok = true;
-    while (ok) {
+Ref<FiftCont> interpret_compile_execute(IntCtx& ctx) {
+  if (ctx.state > 0) {
+    interpret_compile_internal(ctx.stack);
+    return {};
+  } else {
+    return interpret_execute_internal(ctx);
+  }
+}
+
+Ref<FiftCont> InterpretCont::run_tail(IntCtx& ctx) const {
+  if (!ctx.get_input() && !ctx.load_next_line()) {
+    return {};
+  }
+  while (true) {
+    if (!ctx.is_sb()) {
       ctx.skipspc();
-      const char* ptr = ctx.get_input();
-      if (!*ptr) {
+      if (*ctx.get_input()) {
         break;
       }
-      std::string Word;
-      Word.reserve(128);
-      auto entry = ctx.dictionary->lookup("");
-      std::string entry_word;
-      const char* ptr_end = ptr;
-      while (*ptr && *ptr != ' ' && *ptr != '\t') {
-        Word += *ptr++;
-        auto cur = ctx.dictionary->lookup(Word);
-        if (cur) {
-          entry = cur;
-          entry_word = Word;
-          ptr_end = ptr;
-        }
-      }
-      auto cur = ctx.dictionary->lookup(Word + " ");
-      if (cur || !entry) {
-        entry = std::move(cur);
-        ctx.set_input(ptr);
-        ctx.skipspc();
-      } else {
-        Word = entry_word;
-        ctx.set_input(ptr_end);
-      }
-      try {
-        if (entry) {
-          if (entry->is_active()) {
-            (*entry)(ctx);
-          } else {
-            ctx.stack.push_smallint(0);
-            ctx.stack.push({vm::from_object, entry->get_def()});
-          }
-        } else {
-          auto res = numeric_value_ext(Word);
-          ctx.stack.push(std::move(res.first));
-          if (res.second.not_null()) {
-            ctx.stack.push(std::move(res.second));
-            push_argcount(ctx, 2);
-          } else {
-            push_argcount(ctx, 1);
-          }
-        }
-        if (ctx.state > 0) {
-          interpret_compile_internal(ctx.stack);
-        } else {
-          interpret_execute_internal(ctx);
-        }
-      } catch (IntError& ab) {
-        errs << ctx << Word << ": " << ab.msg;
-        ok = false;
-      } catch (vm::VmError& ab) {
-        errs << ctx << Word << ": " << ab.get_msg();
-        ok = false;
-      } catch (vm::CellBuilder::CellWriteError) {
-        errs << ctx << Word << ": Cell builder write error";
-        ok = false;
-      } catch (vm::VmFatal) {
-        errs << ctx << Word << ": fatal vm error";
-        ok = false;
-      } catch (Quit& q) {
-        if (ctx.include_depth) {
-          throw;
-        }
-        if (!q.res) {
-          ok = false;
-        } else {
-          return q.res;
-        }
-      } catch (SkipToEof) {
-        return 0;
-      }
-    };
-    if (!ok) {
-      auto err_msg = errs.str();
-      if (!err_msg.empty()) {
-        LOG(ERROR) << err_msg;
-      }
-      ctx.clear();
-      if (ctx.include_depth) {
-        throw IntError{"error interpreting included file `" + ctx.filename + "` : " + err_msg};
-      }
-    } else if (!ctx.state && !ctx.include_depth) {
+    }
+    if (!ctx.state && !ctx.include_depth) {
       *ctx.output_stream << " ok" << std::endl;
     }
+    if (!ctx.load_next_line()) {
+      return {};
+    }
   }
-  return 0;
+  const char* ptr = ctx.get_input();
+  std::string Word;
+  Word.reserve(128);
+  auto entry = ctx.dictionary->lookup("");
+  std::string entry_word;
+  const char* ptr_end = ptr;
+  while (*ptr && *ptr != ' ' && *ptr != '\t') {
+    Word += *ptr++;
+    auto cur = ctx.dictionary->lookup(Word);
+    if (cur) {
+      entry = cur;
+      entry_word = Word;
+      ptr_end = ptr;
+    }
+  }
+  auto cur = ctx.dictionary->lookup(Word + " ");
+  if (cur || !entry) {
+    entry = std::move(cur);
+    ctx.set_input(ptr);
+    ctx.skipspc();
+  } else {
+    Word = entry_word;
+    ctx.set_input(ptr_end);
+  }
+  ctx.word = Word;
+  static Ref<FiftCont> compile_exec_ref = td::make_ref<CtxTailWord>(interpret_compile_execute);
+  if (!entry) {
+    // numbers
+    auto res = numeric_value_ext(Word);
+    ctx.stack.push(std::move(res.first));
+    if (res.second.not_null()) {
+      ctx.stack.push(std::move(res.second));
+      push_argcount(ctx, 2);
+    } else {
+      push_argcount(ctx, 1);
+    }
+  } else if (!entry->is_active()) {
+    // ordinary word
+    ctx.stack.push_smallint(0);
+    ctx.stack.push({vm::from_object, entry->get_def()});
+  } else {
+    // active word
+    ctx.next = SeqCont::seq(compile_exec_ref, SeqCont::seq(self(), std::move(ctx.next)));
+    return entry->get_def();
+  }
+  exit_interpret->set({vm::from_object, ctx.next});
+  ctx.next = SeqCont::seq(self(), std::move(ctx.next));
+  return compile_exec_ref;
 }
 
 }  // namespace fift

crypto/fift/words.h

@@ -21,23 +21,10 @@
 
 namespace fift {
 
-// thrown by 'quit', 'bye' and 'halt' for exiting to top level
-struct Quit {
-  int res;
-  Quit() : res(0) {
-  }
-  Quit(int _res) : res(_res) {
-  }
-};
-
-struct SkipToEof {};
-
 void init_words_common(Dictionary& dictionary);
 void init_words_vm(Dictionary& dictionary, bool debug_enabled = false);
 void init_words_ton(Dictionary& dictionary);
 
 void import_cmdline_args(Dictionary& d, std::string arg0, int n, const char* const argv[]);
 
-int funny_interpret_loop(IntCtx& ctx);
-
 }  // namespace fift

crypto/func/test/a12.fc

@@ -0,0 +1,4 @@
+_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k, int l, int m, int n, int o) {
+  return (a, i, o, j, e, f, g, h, k, l, m, d, b, c, n);
+  ;; optimal code is 6-byte: s11 s12 XCHG2  2 5 BLKSWAP  s13 s13 s11 XCHG3
+}

crypto/func/test/a12_1.fc

@@ -0,0 +1,3 @@
+_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j) {
+  return (f, j, a, b, i, g, d, e, h, c);
+}

crypto/func/test/a12_2.fc

@@ -0,0 +1,3 @@
+_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j) {
+  return (f, d, j, a, i, g, b, e, h, c);
+}

crypto/func/test/a12_3.fc

@@ -0,0 +1,3 @@
+_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) {
+  return (e, h, d, a, c, g, f, b, i, j, k);   ;; optimal code is 4 ops, 8 bytes
+}

crypto/func/test/a12_4.fc

@@ -0,0 +1,3 @@
+_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) {
+  return (b, a, c, e, g, d, f, k, i, j, h);   ;; optimal code is 4 ops, 8 bytes
+}

crypto/func/test/a12_5.fc

@@ -0,0 +1,3 @@
+_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) {
+  return (c, g, d, k, a, f, e, h, i, j, b);   ;; optimal code is 6 ops, 6 bytes
+}

crypto/func/test/a12_6.fc

@@ -0,0 +1,3 @@
+_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) {
+  return (h, e, d, j, k, f, i, a, b, c, g);   ;; optimal code is 3 ops, 6 bytes
+}

crypto/func/test/a12_7.fc

@@ -0,0 +1,4 @@
+_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k, int l, int m, int n, int o, int p) {
+  return (m, f, l, j, b, e, a, d, p, k, c, n, g, i, h, o);   ;; optimal code is 6 ops, 11 bytes
+  ;; s12 s0 s14 XCHG3  s0 s5 XCHG  6 8 BLKSWAP  s3 s3 s9 XCHG3  s10 s14 s15 XCHG3  s13 s9 s9 XCHG3
+}

crypto/smc-envelope/GenericAccount.cpp

@@ -46,7 +46,8 @@ bool unpack_grams(td::Ref<vm::CellSlice> cs, td::uint64& amount) {
 }
 }  // namespace smc
 
-td::Ref<vm::Cell> GenericAccount::get_init_state(td::Ref<vm::Cell> code, td::Ref<vm::Cell> data) noexcept {
+td::Ref<vm::Cell> GenericAccount::get_init_state(const td::Ref<vm::Cell>& code,
+                                                 const td::Ref<vm::Cell>& data) noexcept {
   return vm::CellBuilder()
       .store_zeroes(2)
       .store_ones(2)
@@ -136,4 +137,23 @@ td::Result<td::Ed25519::PublicKey> GenericAccount::get_public_key(const SmartCon
   };
   return TRY_VM(do_get_public_key());
 }
+
+td::Result<td::uint32> GenericAccount::get_seqno(const SmartContract& sc) {
+  return TRY_VM([&]() -> td::Result<td::uint32> {
+    auto answer = sc.run_get_method("seqno");
+    if (!answer.success) {
+      return td::Status::Error("seqno get method failed");
+    }
+    return static_cast<td::uint32>(answer.stack.write().pop_long_range(std::numeric_limits<td::uint32>::max()));
+  }());
+}
+td::Result<td::uint32> GenericAccount::get_wallet_id(const SmartContract& sc) {
+  return TRY_VM([&]() -> td::Result<td::uint32> {
+    auto answer = sc.run_get_method("wallet_id");
+    if (!answer.success) {
+      return td::Status::Error("seqno get method failed");
+    }
+    return static_cast<td::uint32>(answer.stack.write().pop_long_range(std::numeric_limits<td::uint32>::max()));
+  }());
+}
 }  // namespace ton

crypto/smc-envelope/GenericAccount.h

@@ -29,12 +29,17 @@ bool unpack_grams(td::Ref<vm::CellSlice> cs, td::uint64& amount);
 }  // namespace smc
 class GenericAccount {
  public:
-  static td::Ref<vm::Cell> get_init_state(td::Ref<vm::Cell> code, td::Ref<vm::Cell> data) noexcept;
+  static td::Ref<vm::Cell> get_init_state(const td::Ref<vm::Cell>& code, const td::Ref<vm::Cell>& data) noexcept;
+  static td::Ref<vm::Cell> get_init_state(const SmartContract::State& state) noexcept {
+    return get_init_state(state.code, state.data);
+  }
   static block::StdAddress get_address(ton::WorkchainId workchain_id, const td::Ref<vm::Cell>& init_state) noexcept;
   static td::Ref<vm::Cell> create_ext_message(const block::StdAddress& address, td::Ref<vm::Cell> new_state,
                                               td::Ref<vm::Cell> body) noexcept;
   static void store_int_message(vm::CellBuilder& cb, const block::StdAddress& dest_address, td::int64 gramms);
 
   static td::Result<td::Ed25519::PublicKey> get_public_key(const SmartContract& sc);
+  static td::Result<td::uint32> get_seqno(const SmartContract& sc);
+  static td::Result<td::uint32> get_wallet_id(const SmartContract& sc);
 };
 }  // namespace ton

crypto/smc-envelope/HighloadWallet.cpp

@@ -27,48 +27,12 @@
 #include <limits>
 
 namespace ton {
-td::optional<td::int32> HighloadWallet::guess_revision(const vm::Cell::Hash& code_hash) {
-  for (td::int32 i = 1; i <= 2; i++) {
-    if (get_init_code(i)->get_hash() == code_hash) {
-      return i;
-    }
-  }
-  return {};
-}
-td::optional<td::int32> HighloadWallet::guess_revision(const block::StdAddress& address,
-                                                       const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) {
-  for (td::int32 i = 1; i <= 2; i++) {
-    if (GenericAccount::get_address(address.workchain, get_init_state(public_key, wallet_id, i)) == address) {
-      return i;
-    }
-  }
-  return {};
-}
-td::Ref<vm::Cell> HighloadWallet::get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id,
-                                                 td::int32 revision) noexcept {
-  auto code = get_init_code(revision);
-  auto data = get_init_data(public_key, wallet_id);
-  return GenericAccount::get_init_state(std::move(code), std::move(data));
-}
 
-td::Ref<vm::Cell> HighloadWallet::get_init_message(const td::Ed25519::PrivateKey& private_key,
-                                                   td::uint32 wallet_id) noexcept {
-  td::uint32 seqno = 0;
-  td::uint32 valid_until = std::numeric_limits<td::uint32>::max();
-  auto append_message = [&](auto&& cb) -> vm::CellBuilder& {
-    cb.store_long(wallet_id, 32).store_long(valid_until, 32).store_long(seqno, 32);
-    CHECK(cb.store_maybe_ref({}));
-    return cb;
-  };
-  auto signature = private_key.sign(append_message(vm::CellBuilder()).finalize()->get_hash().as_slice()).move_as_ok();
-
-  return append_message(vm::CellBuilder().store_bytes(signature)).finalize();
-}
-
-td::Ref<vm::Cell> HighloadWallet::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id,
-                                                      td::uint32 seqno, td::uint32 valid_until,
-                                                      td::Span<Gift> gifts) noexcept {
-  CHECK(gifts.size() <= max_gifts_size);
+td::Result<td::Ref<vm::Cell>> HighloadWallet::make_a_gift_message(const td::Ed25519::PrivateKey& private_key,
+                                                                  td::uint32 valid_until, td::Span<Gift> gifts) const {
+  TRY_RESULT(wallet_id, get_wallet_id());
+  TRY_RESULT(seqno, get_seqno());
+  CHECK(gifts.size() <= get_max_gifts_size());
   vm::Dictionary messages(16);
   for (size_t i = 0; i < gifts.size(); i++) {
     auto& gift = gifts[i];
@@ -91,63 +55,36 @@ td::Ref<vm::Cell> HighloadWallet::make_a_gift_message(const td::Ed25519::Private
   return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize();
 }
 
-td::Ref<vm::Cell> HighloadWallet::get_init_code(td::int32 revision) noexcept {
-  return SmartContractCode::get_code(SmartContractCode::HighloadWalletV1, revision);
-}
-
-vm::CellHash HighloadWallet::get_init_code_hash() noexcept {
-  return get_init_code(0)->get_hash();
-}
-
-td::Ref<vm::Cell> HighloadWallet::get_init_data(const td::Ed25519::PublicKey& public_key,
-                                                td::uint32 wallet_id) noexcept {
+td::Ref<vm::Cell> HighloadWallet::get_init_data(const InitData& init_data) noexcept {
   return vm::CellBuilder()
-      .store_long(0, 32)
-      .store_long(wallet_id, 32)
-      .store_bytes(public_key.as_octet_string())
+      .store_long(init_data.seqno, 32)
+      .store_long(init_data.wallet_id, 32)
+      .store_bytes(init_data.public_key)
       .finalize();
 }
 
-td::Result<td::uint32> HighloadWallet::get_seqno() const {
-  return TRY_VM(get_seqno_or_throw());
-}
-
-td::Result<td::uint32> HighloadWallet::get_seqno_or_throw() const {
-  if (state_.data.is_null()) {
-    return 0;
-  }
-  //FIXME use get method
-  return static_cast<td::uint32>(vm::load_cell_slice(state_.data).fetch_ulong(32));
-}
-
 td::Result<td::uint32> HighloadWallet::get_wallet_id() const {
-  return TRY_VM(get_wallet_id_or_throw());
-}
-
-td::Result<td::uint32> HighloadWallet::get_wallet_id_or_throw() const {
-  if (state_.data.is_null()) {
-    return 0;
-  }
-  //FIXME use get method
-  auto cs = vm::load_cell_slice(state_.data);
-  cs.skip_first(32);
-  return static_cast<td::uint32>(cs.fetch_ulong(32));
+  return TRY_VM([&]() -> td::Result<td::uint32> {
+    if (state_.data.is_null()) {
+      return 0;
+    }
+    auto cs = vm::load_cell_slice(state_.data);
+    cs.skip_first(32);
+    return static_cast<td::uint32>(cs.fetch_ulong(32));
+  }());
 }
 
 td::Result<td::Ed25519::PublicKey> HighloadWallet::get_public_key() const {
-  return TRY_VM(get_public_key_or_throw());
-}
-
-td::Result<td::Ed25519::PublicKey> HighloadWallet::get_public_key_or_throw() const {
-  if (state_.data.is_null()) {
-    return td::Status::Error("data is null");
-  }
-  //FIXME use get method
-  auto cs = vm::load_cell_slice(state_.data);
-  cs.skip_first(64);
-  td::SecureString res(td::Ed25519::PublicKey::LENGTH);
-  cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast<td::int32>(res.size()));
-  return td::Ed25519::PublicKey(std::move(res));
+  return TRY_VM([&]() -> td::Result<td::Ed25519::PublicKey> {
+    if (state_.data.is_null()) {
+      return td::Status::Error("data is null");
+    }
+    auto cs = vm::load_cell_slice(state_.data);
+    cs.skip_first(64);
+    td::SecureString res(td::Ed25519::PublicKey::LENGTH);
+    cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast<td::int32>(res.size()));
+    return td::Ed25519::PublicKey(std::move(res));
+  }());
 }
 
 }  // namespace ton

crypto/smc-envelope/HighloadWallet.h

@@ -26,41 +26,24 @@
 #include "vm/cells/CellString.h"
 
 namespace ton {
-class HighloadWallet : public ton::SmartContract, public WalletInterface {
- public:
-  explicit HighloadWallet(State state) : ton::SmartContract(std::move(state)) {
-  }
+struct HighloadWalletTraits {
+  using InitData = WalletInterface::DefaultInitData;
+
   static constexpr unsigned max_message_size = vm::CellString::max_bytes;
   static constexpr unsigned max_gifts_size = 254;
-  static td::Ref<vm::Cell> get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id,
-                                          td::int32 revision) noexcept;
-  static td::Ref<vm::Cell> get_init_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id) noexcept;
-  static td::Ref<vm::Cell> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id,
-                                               td::uint32 seqno, td::uint32 valid_until, td::Span<Gift> gifts) noexcept;
-
-  static td::Ref<vm::Cell> get_init_code(td::int32 revision) noexcept;
-  static vm::CellHash get_init_code_hash() noexcept;
-  static td::Ref<vm::Cell> get_init_data(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) noexcept;
-  static td::optional<td::int32> guess_revision(const vm::Cell::Hash& code_hash);
-  static td::optional<td::int32> guess_revision(const block::StdAddress& address,
-                                                const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id);
-  td::Result<td::uint32> get_seqno() const;
-  td::Result<td::uint32> get_wallet_id() const;
+  static constexpr auto code_type = SmartContractCode::HighloadWalletV1;
+};
 
+class HighloadWallet : public WalletBase<HighloadWallet, HighloadWalletTraits> {
+ public:
+  explicit HighloadWallet(State state) : WalletBase(std::move(state)) {
+  }
   td::Result<td::Ref<vm::Cell>> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until,
-                                                    td::Span<Gift> gifts) const override {
-    TRY_RESULT(seqno, get_seqno());
-    TRY_RESULT(wallet_id, get_wallet_id());
-    return make_a_gift_message(private_key, wallet_id, seqno, valid_until, gifts);
-  }
-  size_t get_max_gifts_size() const override {
-    return max_gifts_size;
-  }
-  td::Result<td::Ed25519::PublicKey> get_public_key() const override;
+                                                    td::Span<Gift> gifts) const override;
+  static td::Ref<vm::Cell> get_init_data(const InitData& init_data) noexcept;
 
- private:
-  td::Result<td::uint32> get_seqno_or_throw() const;
-  td::Result<td::uint32> get_wallet_id_or_throw() const;
-  td::Result<td::Ed25519::PublicKey> get_public_key_or_throw() const;
+  // can't use get methods for compatibility with old revisions
+  td::Result<td::uint32> get_wallet_id() const override;
+  td::Result<td::Ed25519::PublicKey> get_public_key() const override;
 };
 }  // namespace ton

crypto/smc-envelope/HighloadWalletV2.cpp

@@ -27,33 +27,10 @@
 #include <limits>
 
 namespace ton {
-td::optional<td::int32> HighloadWalletV2::guess_revision(const vm::Cell::Hash& code_hash) {
-  for (td::int32 i = 1; i <= 2; i++) {
-    if (get_init_code(i)->get_hash() == code_hash) {
-      return i;
-    }
-  }
-  return {};
-}
-td::optional<td::int32> HighloadWalletV2::guess_revision(const block::StdAddress& address,
-                                                         const td::Ed25519::PublicKey& public_key,
-                                                         td::uint32 wallet_id) {
-  for (td::int32 i = 1; i <= 2; i++) {
-    if (GenericAccount::get_address(address.workchain, get_init_state(public_key, wallet_id, i)) == address) {
-      return i;
-    }
-  }
-  return {};
-}
-td::Ref<vm::Cell> HighloadWalletV2::get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id,
-                                                   td::int32 revision) noexcept {
-  auto code = get_init_code(revision);
-  auto data = get_init_data(public_key, wallet_id);
-  return GenericAccount::get_init_state(std::move(code), std::move(data));
-}
 
-td::Ref<vm::Cell> HighloadWalletV2::get_init_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id,
-                                                     td::uint32 valid_until) noexcept {
+td::Result<td::Ref<vm::Cell>> HighloadWalletV2::get_init_message(const td::Ed25519::PrivateKey& private_key,
+                                                                 td::uint32 valid_until) const noexcept {
+  TRY_RESULT(wallet_id, get_wallet_id());
   td::uint32 id = -1;
   auto append_message = [&](auto&& cb) -> vm::CellBuilder& {
     cb.store_long(wallet_id, 32).store_long(valid_until, 32).store_long(id, 32);
@@ -65,10 +42,11 @@ td::Ref<vm::Cell> HighloadWalletV2::get_init_message(const td::Ed25519::PrivateK
   return append_message(vm::CellBuilder().store_bytes(signature)).finalize();
 }
 
-td::Ref<vm::Cell> HighloadWalletV2::make_a_gift_message(const td::Ed25519::PrivateKey& private_key,
-                                                        td::uint32 wallet_id, td::uint32 valid_until,
-                                                        td::Span<Gift> gifts) noexcept {
-  CHECK(gifts.size() <= max_gifts_size);
+td::Result<td::Ref<vm::Cell>> HighloadWalletV2::make_a_gift_message(const td::Ed25519::PrivateKey& private_key,
+                                                                    td::uint32 valid_until,
+                                                                    td::Span<Gift> gifts) const {
+  TRY_RESULT(wallet_id, get_wallet_id());
+  CHECK(gifts.size() <= get_max_gifts_size());
   vm::Dictionary messages(16);
   for (size_t i = 0; i < gifts.size(); i++) {
     auto& gift = gifts[i];
@@ -96,49 +74,34 @@ td::Ref<vm::Cell> HighloadWalletV2::make_a_gift_message(const td::Ed25519::Priva
   return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize();
 }
 
-td::Ref<vm::Cell> HighloadWalletV2::get_init_code(td::int32 revision) noexcept {
-  return SmartContractCode::get_code(SmartContractCode::HighloadWalletV2, revision);
-}
-
-vm::CellHash HighloadWalletV2::get_init_code_hash() noexcept {
-  return get_init_code(0)->get_hash();
-}
-
-td::Ref<vm::Cell> HighloadWalletV2::get_init_data(const td::Ed25519::PublicKey& public_key,
-                                                  td::uint32 wallet_id) noexcept {
+td::Ref<vm::Cell> HighloadWalletV2::get_init_data(const InitData& init_data) noexcept {
   vm::CellBuilder cb;
-  cb.store_long(wallet_id, 32).store_long(0, 64).store_bytes(public_key.as_octet_string());
+  cb.store_long(init_data.wallet_id, 32).store_long(init_data.seqno, 64).store_bytes(init_data.public_key);
   CHECK(cb.store_maybe_ref({}));
   return cb.finalize();
 }
 
 td::Result<td::uint32> HighloadWalletV2::get_wallet_id() const {
-  return TRY_VM(get_wallet_id_or_throw());
-}
-
-td::Result<td::uint32> HighloadWalletV2::get_wallet_id_or_throw() const {
-  if (state_.data.is_null()) {
-    return 0;
-  }
-  //FIXME use get method
-  auto cs = vm::load_cell_slice(state_.data);
-  return static_cast<td::uint32>(cs.fetch_ulong(32));
+  return TRY_VM([&]() -> td::Result<td::uint32> {
+    if (state_.data.is_null()) {
+      return 0;
+    }
+    auto cs = vm::load_cell_slice(state_.data);
+    return static_cast<td::uint32>(cs.fetch_ulong(32));
+  }());
 }
 
 td::Result<td::Ed25519::PublicKey> HighloadWalletV2::get_public_key() const {
-  return TRY_VM(get_public_key_or_throw());
-}
-
-td::Result<td::Ed25519::PublicKey> HighloadWalletV2::get_public_key_or_throw() const {
-  if (state_.data.is_null()) {
-    return td::Status::Error("data is null");
-  }
-  //FIXME use get method
-  auto cs = vm::load_cell_slice(state_.data);
-  cs.skip_first(96);
-  td::SecureString res(td::Ed25519::PublicKey::LENGTH);
-  cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast<td::int32>(res.size()));
-  return td::Ed25519::PublicKey(std::move(res));
+  return TRY_VM([&]() -> td::Result<td::Ed25519::PublicKey> {
+    if (state_.data.is_null()) {
+      return td::Status::Error("data is null");
+    }
+    auto cs = vm::load_cell_slice(state_.data);
+    cs.skip_first(96);
+    td::SecureString res(td::Ed25519::PublicKey::LENGTH);
+    cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast<td::int32>(res.size()));
+    return td::Ed25519::PublicKey(std::move(res));
+  }());
 }
 
 }  // namespace ton

crypto/smc-envelope/HighloadWalletV2.h

@@ -26,41 +26,26 @@
 #include "vm/cells/CellString.h"
 
 namespace ton {
-class HighloadWalletV2 : public ton::SmartContract, public WalletInterface {
- public:
-  explicit HighloadWalletV2(State state) : ton::SmartContract(std::move(state)) {
-  }
+struct HighloadWalletV2Traits {
+  using InitData = WalletInterface::DefaultInitData;
+
   static constexpr unsigned max_message_size = vm::CellString::max_bytes;
   static constexpr unsigned max_gifts_size = 254;
-  static td::Ref<vm::Cell> get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id,
-                                          td::int32 revision) noexcept;
-  static td::Ref<vm::Cell> get_init_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id,
-                                            td::uint32 valid_until) noexcept;
-
-  static td::Ref<vm::Cell> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id,
-                                               td::uint32 valid_until, td::Span<Gift> gifts) noexcept;
-
-  static td::Ref<vm::Cell> get_init_code(td::int32 revision) noexcept;
-  static vm::CellHash get_init_code_hash() noexcept;
-  static td::Ref<vm::Cell> get_init_data(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) noexcept;
-  static td::optional<td::int32> guess_revision(const vm::Cell::Hash& code_hash);
-  static td::optional<td::int32> guess_revision(const block::StdAddress& address,
-                                                const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id);
-
-  td::Result<td::uint32> get_wallet_id() const;
-
+  static constexpr auto code_type = SmartContractCode::HighloadWalletV2;
+};
+class HighloadWalletV2 : public WalletBase<HighloadWalletV2, HighloadWalletV2Traits> {
+ public:
+  explicit HighloadWalletV2(State state) : WalletBase(std::move(state)) {
+  }
   td::Result<td::Ref<vm::Cell>> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until,
-                                                    td::Span<Gift> gifts) const override {
-    TRY_RESULT(wallet_id, get_wallet_id());
-    return make_a_gift_message(private_key, wallet_id, valid_until, gifts);
-  }
-  size_t get_max_gifts_size() const override {
-    return max_gifts_size;
-  }
-  td::Result<td::Ed25519::PublicKey> get_public_key() const override;
+                                                    td::Span<Gift> gifts) const override;
+  static td::Ref<vm::Cell> get_init_data(const InitData& init_data) noexcept;
+  td::Result<td::Ref<vm::Cell>> get_init_message(const td::Ed25519::PrivateKey& private_key,
+                                                 td::uint32 valid_until = std::numeric_limits<td::uint32>::max()) const
+      noexcept;
 
- private:
-  td::Result<td::uint32> get_wallet_id_or_throw() const;
-  td::Result<td::Ed25519::PublicKey> get_public_key_or_throw() const;
+  // can't use get methods for compatibility with old revisions
+  td::Result<td::uint32> get_wallet_id() const override;
+  td::Result<td::Ed25519::PublicKey> get_public_key() const override;
 };
 }  // namespace ton

crypto/smc-envelope/SmartContract.cpp

@@ -30,6 +30,14 @@
 #include "td/utils/crypto.h"
 
 namespace ton {
+int SmartContract::Answer::output_actions_count(td::Ref<vm::Cell> list) {
+  int i = -1;
+  do {
+    ++i;
+    list = load_cell_slice(std::move(list)).prefetch_ref();
+  } while (list.not_null());
+  return i;
+}
 namespace {
 
 td::Ref<vm::Stack> prepare_vm_stack(td::RefInt256 amount, td::Ref<vm::CellSlice> body) {
@@ -66,15 +74,6 @@ td::Ref<vm::Tuple> prepare_vm_c7(td::uint32 now, td::uint64 balance) {
   return vm::make_tuple_ref(std::move(tuple));
 }
 
-static int output_actions_count(td::Ref<vm::Cell> list) {
-  int i = -1;
-  do {
-    ++i;
-    list = load_cell_slice(std::move(list)).prefetch_ref();
-  } while (list.not_null());
-  return i;
-}
-
 SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref<vm::Stack> stack, td::Ref<vm::Tuple> c7,
                                     vm::GasLimits gas, bool ignore_chksig) {
   auto gas_credit = gas.gas_credit;
@@ -133,7 +132,7 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref<vm::Stac
     res.new_state.data = vm.get_c4();
     res.actions = vm.get_d(5);
     LOG(DEBUG) << "output actions:\n"
-               << block::gen::OutList{output_actions_count(res.actions)}.as_string_ref(res.actions);
+               << block::gen::OutList{res.output_actions_count(res.actions)}.as_string_ref(res.actions);
   }
   LOG_IF(ERROR, gas_credit != 0 && (res.accepted && !res.success))
       << "Accepted but failed with code " << res.code << "\n"

crypto/smc-envelope/SmartContract.h

@@ -48,6 +48,7 @@ class SmartContract : public td::CntObject {
     td::Ref<vm::Cell> actions;
     td::int32 code;
     td::int64 gas_used;
+    static int output_actions_count(td::Ref<vm::Cell> list);
   };
 
   struct Args {

crypto/smc-envelope/SmartContractCode.cpp

@@ -38,8 +38,6 @@ const auto& get_map() {
       map[name] = vm::std_boc_deserialize(td::base64_decode(code_str).move_as_ok()).move_as_ok();
     };
 #include "smartcont/auto/multisig-code.cpp"
-#include "smartcont/auto/simple-wallet-ext-code.cpp"
-#include "smartcont/auto/simple-wallet-code.cpp"
 #include "smartcont/auto/wallet-code.cpp"
 #include "smartcont/auto/highload-wallet-code.cpp"
 #include "smartcont/auto/highload-wallet-v2-code.cpp"
@@ -65,18 +63,6 @@ const auto& get_map() {
                   "QE0VNggED0Dm+hMfJgUXO68qIH+QFUEIf5EPKjAvQE0fgAf44WIYAQ9HhvpSCYAtMH1DAB+wCRMuIBs+"
                   "ZbgyWhyEA0gED0Q4rmMcgSyx8Tyz/L//QAye1UCAAE0DACASAGBwAXvZznaiaGmvmOuF/8AEG+X5dqJoaY+Y6Z/p/"
                   "5j6AmipEEAgegc30JjJLb/JXdHxQANCCAQPSWb6UyURCUMFMDud4gkzM2AZIyMOKz");
-    with_tvm_code("simple-wallet-r1",
-                  "te6ccgEEAQEAAAAAUwAAov8AIN0gggFMl7qXMO1E0NcLH+Ck8mCBAgDXGCDXCx/tRNDTH9P/"
-                  "0VESuvKhIvkBVBBE+RDyovgAAdMfMSDXSpbTB9QC+wDe0aTIyx/L/8ntVA==");
-    with_tvm_code("simple-wallet-r2",
-                  "te6ccgEBAQEAXwAAuv8AIN0gggFMl7ohggEznLqxnHGw7UTQ0x/XC//jBOCk8mCBAgDXGCDXCx/tRNDTH9P/"
-                  "0VESuvKhIvkBVBBE+RDyovgAAdMfMSDXSpbTB9QC+wDe0aTIyx/L/8ntVA==");
-    with_tvm_code("wallet-r1",
-                  "te6ccgEBAQEAVwAAqv8AIN0gggFMl7qXMO1E0NcLH+Ck8mCDCNcYINMf0x8B+CO78mPtRNDTH9P/0VExuvKhA/"
-                  "kBVBBC+RDyovgAApMg10qW0wfUAvsA6NGkyMsfy//J7VQ=");
-    with_tvm_code("wallet-r2",
-                  "te6ccgEBAQEAYwAAwv8AIN0gggFMl7ohggEznLqxnHGw7UTQ0x/XC//jBOCk8mCDCNcYINMf0x8B+CO78mPtRNDTH9P/"
-                  "0VExuvKhA/kBVBBC+RDyovgAApMg10qW0wfUAvsA6NGkyMsfy//J7VQ=");
     with_tvm_code("wallet3-r1",
                   "te6ccgEBAQEAYgAAwP8AIN0gggFMl7qXMO1E0NcLH+Ck8mCDCNcYINMf0x/TH/gjE7vyY+1E0NMf0x/T/"
                   "9FRMrryoVFEuvKiBPkBVBBV+RDyo/gAkyDXSpbTB9QC+wDo0QGkyMsfyx/L/8ntVA==");
@@ -123,22 +109,10 @@ td::Result<td::Ref<vm::Cell>> SmartContractCode::load(td::Slice name) {
 
 td::Span<int> SmartContractCode::get_revisions(Type type) {
   switch (type) {
-    case Type::WalletV1: {
-      static int res[] = {1, 2};
-      return res;
-    }
-    case Type::WalletV2: {
-      static int res[] = {1, 2};
-      return res;
-    }
     case Type::WalletV3: {
       static int res[] = {1, 2};
       return res;
     }
-    case Type::WalletV1Ext: {
-      static int res[] = {-1};
-      return res;
-    }
     case Type::HighloadWalletV1: {
       static int res[] = {-1, 1, 2};
       return res;
@@ -191,14 +165,8 @@ td::Ref<vm::Cell> SmartContractCode::get_code(Type type, int ext_revision) {
   auto revision = validate_revision(type, ext_revision).move_as_ok();
   auto basename = [](Type type) -> td::Slice {
     switch (type) {
-      case Type::WalletV1:
-        return "simple-wallet";
-      case Type::WalletV2:
-        return "wallet";
       case Type::WalletV3:
         return "wallet3";
-      case Type::WalletV1Ext:
-        return "simple-wallet-ext";
       case Type::HighloadWalletV1:
         return "highload-wallet";
       case Type::HighloadWalletV2:

crypto/smc-envelope/SmartContractCode.h

@@ -26,18 +26,7 @@ class SmartContractCode {
  public:
   static td::Result<td::Ref<vm::Cell>> load(td::Slice name);
 
-  enum Type {
-    WalletV1 = 1,
-    WalletV1Ext,
-    WalletV2,
-    WalletV3,
-    HighloadWalletV1,
-    HighloadWalletV2,
-    ManualDns,
-    Multisig,
-    PaymentChannel,
-    RestrictedWallet
-  };
+  enum Type { WalletV3 = 4, HighloadWalletV1, HighloadWalletV2, ManualDns, Multisig, PaymentChannel, RestrictedWallet };
   static td::Span<int> get_revisions(Type type);
   static td::Result<int> validate_revision(Type type, int revision);
   static td::Ref<vm::Cell> get_code(Type type, int revision = 0);

crypto/smc-envelope/WalletInterface.cpp

@@ -0,0 +1,77 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2020 Telegram Systems LLP
+*/
+#include "WalletInterface.h"
+
+namespace ton {
+td::Result<td::uint64> WalletInterface::get_balance(td::uint64 account_balance, td::uint32 now) const {
+  return TRY_VM([&]() -> td::Result<td::uint64> {
+    Answer answer = this->run_get_method(Args().set_method_id("balance").set_balance(account_balance).set_now(now));
+    if (!answer.success) {
+      return td::Status::Error("balance get method failed");
+    }
+    return static_cast<td::uint64>(answer.stack.write().pop_long());
+  }());
+}
+
+td::Result<td::Ed25519::PublicKey> WalletInterface::get_public_key() const {
+  return GenericAccount::get_public_key(*this);
+};
+
+td::Result<td::uint32> WalletInterface::get_seqno() const {
+  return GenericAccount::get_seqno(*this);
+}
+
+td::Result<td::uint32> WalletInterface::get_wallet_id() const {
+  return GenericAccount::get_wallet_id(*this);
+}
+
+td::Result<td::Ref<vm::Cell>> WalletInterface::get_init_message(const td::Ed25519::PrivateKey &private_key,
+                                                                td::uint32 valid_until) const {
+  return make_a_gift_message(private_key, valid_until, {});
+}
+
+td::Ref<vm::Cell> WalletInterface::create_int_message(const Gift &gift) {
+  vm::CellBuilder cbi;
+  GenericAccount::store_int_message(cbi, gift.destination, gift.gramms < 0 ? 0 : gift.gramms);
+  if (gift.init_state.not_null()) {
+    cbi.store_ones(2);
+    cbi.store_ref(gift.init_state);
+  } else {
+    cbi.store_zeroes(1);
+  }
+  cbi.store_zeroes(1);
+  store_gift_message(cbi, gift);
+  return cbi.finalize();
+}
+void WalletInterface::store_gift_message(vm::CellBuilder &cb, const Gift &gift) {
+  if (gift.body.not_null()) {
+    auto body = vm::load_cell_slice(gift.body);
+    //TODO: handle error
+    CHECK(cb.append_cellslice_bool(body));
+    return;
+  }
+
+  if (gift.is_encrypted) {
+    cb.store_long(1, 32);
+  } else {
+    cb.store_long(0, 32);
+  }
+  vm::CellString::store(cb, gift.message, 35 * 8).ensure();
+}
+}  // namespace ton

crypto/smc-envelope/WalletInterface.h

@@ -21,13 +21,17 @@
 #include "td/utils/common.h"
 #include "Ed25519.h"
 #include "block/block.h"
+#include "block/block-parse.h"
 #include "vm/cells/CellString.h"
 
 #include "SmartContract.h"
+#include "SmartContractCode.h"
 #include "GenericAccount.h"
 
+#include <algorithm>
+
 namespace ton {
-class WalletInterface {
+class WalletInterface : public SmartContract {
  public:
   struct Gift {
     block::StdAddress destination;
@@ -39,49 +43,91 @@ class WalletInterface {
     td::Ref<vm::Cell> body;
     td::Ref<vm::Cell> init_state;
   };
+  struct DefaultInitData {
+    td::SecureString public_key;
+    td::uint32 wallet_id{0};
+    td::uint32 seqno{0};
+    DefaultInitData() = default;
+    DefaultInitData(td::Slice key, td::uint32 wallet_id) : public_key(key), wallet_id(wallet_id) {
+    }
+  };
+
+  WalletInterface(State state) : SmartContract(std::move(state)) {
+  }
 
   virtual ~WalletInterface() {
   }
 
   virtual size_t get_max_gifts_size() const = 0;
+  virtual size_t get_max_message_size() const = 0;
   virtual td::Result<td::Ref<vm::Cell>> make_a_gift_message(const td::Ed25519::PrivateKey &private_key,
                                                             td::uint32 valid_until, td::Span<Gift> gifts) const = 0;
-  virtual td::Result<td::Ed25519::PublicKey> get_public_key() const {
-    return td::Status::Error("Unsupported");
+
+  virtual td::Result<td::uint32> get_seqno() const;
+  virtual td::Result<td::uint32> get_wallet_id() const;
+  virtual td::Result<td::uint64> get_balance(td::uint64 account_balance, td::uint32 now) const;
+  virtual td::Result<td::Ed25519::PublicKey> get_public_key() const;
+
+  td::Result<td::Ref<vm::Cell>> get_init_message(const td::Ed25519::PrivateKey &private_key,
+                                                 td::uint32 valid_until = std::numeric_limits<td::uint32>::max()) const;
+
+  static td::Ref<vm::Cell> create_int_message(const Gift &gift);
+  static void store_gift_message(vm::CellBuilder &cb, const Gift &gift);
+};
+
+template <class WalletT, class TraitsT>
+class WalletBase : public WalletInterface {
+ public:
+  using Traits = TraitsT;
+  using InitData = typename Traits::InitData;
+
+  explicit WalletBase(State state) : WalletInterface(std::move(state)) {
   }
 
-  td::Result<td::Ref<vm::Cell>> get_init_message(
-      const td::Ed25519::PrivateKey &private_key,
-      td::uint32 valid_until = std::numeric_limits<td::uint32>::max()) const {
-    return make_a_gift_message(private_key, valid_until, {});
+  size_t get_max_gifts_size() const override {
+    return Traits::max_gifts_size;
   }
-  static td::Ref<vm::Cell> create_int_message(const Gift &gift) {
-    vm::CellBuilder cbi;
-    GenericAccount::store_int_message(cbi, gift.destination, gift.gramms < 0 ? 0 : gift.gramms);
-    if (gift.init_state.not_null()) {
-      cbi.store_ones(2);
-      cbi.store_ref(gift.init_state);
-    } else {
-      cbi.store_zeroes(1);
-    }
-    cbi.store_zeroes(1);
-    store_gift_message(cbi, gift);
-    return cbi.finalize();
+  size_t get_max_message_size() const override {
+    return Traits::max_message_size;
   }
-  static void store_gift_message(vm::CellBuilder &cb, const Gift &gift) {
-    if (gift.body.not_null()) {
-      auto body = vm::load_cell_slice(gift.body);
-      //TODO: handle error
-      CHECK(cb.append_cellslice_bool(body));
-      return;
-    }
 
-    if (gift.is_encrypted) {
-      cb.store_long(1, 32);
-    } else {
-      cb.store_long(0, 32);
+  static td::Ref<WalletT> create(State state) {
+    return td::Ref<WalletT>(true, std::move(state));
+  }
+  static td::Ref<vm::Cell> get_init_code(int revision) {
+    return SmartContractCode::get_code(get_code_type(), revision);
+  };
+  static State get_init_state(int revision, const InitData &init_data) {
+    return {get_init_code(revision), WalletT::get_init_data(init_data)};
+  }
+  static SmartContractCode::Type get_code_type() {
+    return Traits::code_type;
+  }
+  static td::optional<td::int32> guess_revision(const vm::Cell::Hash &code_hash) {
+    for (auto revision : ton::SmartContractCode::get_revisions(get_code_type())) {
+      auto code = get_init_code(revision);
+      if (code->get_hash() == code_hash) {
+        return revision;
+      }
     }
-    vm::CellString::store(cb, gift.message, 35 * 8).ensure();
+    return {};
+  }
+  static td::Span<td::int32> get_revisions() {
+    return ton::SmartContractCode::get_revisions(get_code_type());
+  }
+  static td::optional<td::int32> guess_revision(block::StdAddress &address, const InitData &init_data) {
+    for (auto revision : get_revisions()) {
+      if (WalletT(get_init_state(revision, init_data)).get_address(address.workchain) == address) {
+        return revision;
+      }
+    }
+    return {};
+  }
+  static td::Ref<WalletT> create(const InitData &init_data, int revision) {
+    return td::Ref<WalletT>(true, State{get_init_code(revision), WalletT::get_init_data(init_data)});
+  }
+  CntObject *make_copy() const override {
+    return new WalletT(get_state());
   }
 };
 

crypto/smc-envelope/WalletV3.cpp

@@ -27,36 +27,11 @@
 #include <limits>
 
 namespace ton {
-td::Ref<vm::Cell> WalletV3::get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id,
-                                           td::int32 revision) noexcept {
-  auto code = get_init_code(revision);
-  auto data = get_init_data(public_key, wallet_id);
-  return GenericAccount::get_init_state(std::move(code), std::move(data));
-}
-
-td::optional<td::int32> WalletV3::guess_revision(const vm::Cell::Hash& code_hash) {
-  for (td::int32 i = 1; i <= 2; i++) {
-    if (get_init_code(i)->get_hash() == code_hash) {
-      return i;
-    }
-  }
-  return {};
-}
-td::optional<td::int32> WalletV3::guess_revision(const block::StdAddress& address,
-                                                 const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) {
-  for (td::int32 i = 1; i <= 2; i++) {
-    if (GenericAccount::get_address(address.workchain, get_init_state(public_key, wallet_id, i)) == address) {
-      return i;
-    }
-  }
-  return {};
-}
-
-td::Ref<vm::Cell> WalletV3::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id,
-                                                td::uint32 seqno, td::uint32 valid_until,
-                                                td::Span<Gift> gifts) noexcept {
-  CHECK(gifts.size() <= max_gifts_size);
-
+td::Result<td::Ref<vm::Cell>> WalletV3::make_a_gift_message(const td::Ed25519::PrivateKey& private_key,
+                                                            td::uint32 valid_until, td::Span<Gift> gifts) const {
+  CHECK(gifts.size() <= get_max_gifts_size());
+  TRY_RESULT(seqno, get_seqno());
+  TRY_RESULT(wallet_id, get_wallet_id());
   vm::CellBuilder cb;
   cb.store_long(wallet_id, 32).store_long(valid_until, 32).store_long(seqno, 32);
 
@@ -73,63 +48,36 @@ td::Ref<vm::Cell> WalletV3::make_a_gift_message(const td::Ed25519::PrivateKey& p
   return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize();
 }
 
-td::Ref<vm::Cell> WalletV3::get_init_code(td::int32 revision) noexcept {
-  return SmartContractCode::get_code(ton::SmartContractCode::WalletV3, revision);
-}
-
-vm::CellHash WalletV3::get_init_code_hash() noexcept {
-  return get_init_code()->get_hash();
-}
-
-td::Ref<vm::Cell> WalletV3::get_init_data(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id,
-                                          td::uint32 seqno) noexcept {
+td::Ref<vm::Cell> WalletV3::get_init_data(const InitData& init_data) noexcept {
   return vm::CellBuilder()
-      .store_long(seqno, 32)
-      .store_long(wallet_id, 32)
-      .store_bytes(public_key.as_octet_string())
+      .store_long(init_data.seqno, 32)
+      .store_long(init_data.wallet_id, 32)
+      .store_bytes(init_data.public_key)
       .finalize();
 }
 
-td::Result<td::uint32> WalletV3::get_seqno() const {
-  return TRY_VM(get_seqno_or_throw());
-}
-
-td::Result<td::uint32> WalletV3::get_seqno_or_throw() const {
-  if (state_.data.is_null()) {
-    return 0;
-  }
-  //FIXME use get method
-  return static_cast<td::uint32>(vm::load_cell_slice(state_.data).fetch_ulong(32));
-}
-
 td::Result<td::uint32> WalletV3::get_wallet_id() const {
-  return TRY_VM(get_wallet_id_or_throw());
-}
-
-td::Result<td::uint32> WalletV3::get_wallet_id_or_throw() const {
-  if (state_.data.is_null()) {
-    return 0;
-  }
-  //FIXME use get method
-  auto cs = vm::load_cell_slice(state_.data);
-  cs.skip_first(32);
-  return static_cast<td::uint32>(cs.fetch_ulong(32));
+  return TRY_VM([&]() -> td::Result<td::uint32> {
+    if (state_.data.is_null()) {
+      return 0;
+    }
+    auto cs = vm::load_cell_slice(state_.data);
+    cs.skip_first(32);
+    return static_cast<td::uint32>(cs.fetch_ulong(32));
+  }());
 }
 
 td::Result<td::Ed25519::PublicKey> WalletV3::get_public_key() const {
-  return TRY_VM(get_public_key_or_throw());
-}
-
-td::Result<td::Ed25519::PublicKey> WalletV3::get_public_key_or_throw() const {
-  if (state_.data.is_null()) {
-    return td::Status::Error("data is null");
-  }
-  //FIXME use get method
-  auto cs = vm::load_cell_slice(state_.data);
-  cs.skip_first(64);
-  td::SecureString res(td::Ed25519::PublicKey::LENGTH);
-  cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast<td::int32>(res.size()));
-  return td::Ed25519::PublicKey(std::move(res));
+  return TRY_VM([&]() -> td::Result<td::Ed25519::PublicKey> {
+    if (state_.data.is_null()) {
+      return td::Status::Error("data is null");
+    }
+    auto cs = vm::load_cell_slice(state_.data);
+    cs.skip_first(64);
+    td::SecureString res(td::Ed25519::PublicKey::LENGTH);
+    cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast<td::int32>(res.size()));
+    return td::Ed25519::PublicKey(std::move(res));
+  }());
 }
 
 }  // namespace ton

crypto/smc-envelope/WalletV3.h

@@ -26,135 +26,30 @@
 #include "vm/cells/CellString.h"
 
 namespace ton {
-class WalletV3 : public ton::SmartContract, public WalletInterface {
- public:
-  explicit WalletV3(State state) : ton::SmartContract(std::move(state)) {
-  }
-  explicit WalletV3(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id, td::uint32 seqno = 0)
-      : WalletV3(State{get_init_code(), get_init_data(public_key, wallet_id, seqno)}) {
-  }
+
+struct WalletV3Traits {
+  using InitData = WalletInterface::DefaultInitData;
+
   static constexpr unsigned max_message_size = vm::CellString::max_bytes;
   static constexpr unsigned max_gifts_size = 4;
+  static constexpr auto code_type = SmartContractCode::WalletV3;
+};
 
-  static td::optional<td::int32> guess_revision(const vm::Cell::Hash& code_hash);
-  static td::optional<td::int32> guess_revision(const block::StdAddress& address,
-                                                const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id);
-  static td::Ref<vm::Cell> get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id,
-                                          td::int32 revision = 0) noexcept;
-  static td::Ref<vm::Cell> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id,
-                                               td::uint32 seqno, td::uint32 valid_until, td::Span<Gift> gifts) noexcept;
-
-  static td::Ref<vm::Cell> get_init_code(td::int32 revision = 0) noexcept;
-  static vm::CellHash get_init_code_hash() noexcept;
-  static td::Ref<vm::Cell> get_init_data(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id,
-                                         td::uint32 seqno = 0) noexcept;
-
-  td::Result<td::uint32> get_seqno() const;
-  td::Result<td::uint32> get_wallet_id() const;
-
-  using WalletInterface::get_init_message;
+class WalletV3 : public WalletBase<WalletV3, WalletV3Traits> {
+ public:
+  explicit WalletV3(State state) : WalletBase(std::move(state)) {
+  }
   td::Result<td::Ref<vm::Cell>> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until,
-                                                    td::Span<Gift> gifts) const override {
-    TRY_RESULT(seqno, get_seqno());
-    TRY_RESULT(wallet_id, get_wallet_id());
-    return make_a_gift_message(private_key, wallet_id, seqno, valid_until, gifts);
-  }
-  size_t get_max_gifts_size() const override {
-    return max_gifts_size;
-  }
-  td::Result<td::Ed25519::PublicKey> get_public_key() const override;
+                                                    td::Span<Gift> gifts) const override;
+  static td::Ref<vm::Cell> get_init_data(const InitData& init_data) noexcept;
 
- private:
-  td::Result<td::uint32> get_seqno_or_throw() const;
-  td::Result<td::uint32> get_wallet_id_or_throw() const;
-  td::Result<td::Ed25519::PublicKey> get_public_key_or_throw() const;
+  // can't use get methods for compatibility with old revisions
+  td::Result<td::uint32> get_wallet_id() const override;
+  td::Result<td::Ed25519::PublicKey> get_public_key() const override;
 };
 }  // namespace ton
 
-#include "smc-envelope/SmartContractCode.h"
-#include "smc-envelope/GenericAccount.h"
-#include "block/block-parse.h"
-#include <algorithm>
 namespace ton {
-template <class WalletT, class TraitsT>
-class WalletBase : public SmartContract, public WalletInterface {
- public:
-  using Traits = TraitsT;
-  using InitData = typename Traits::InitData;
-
-  explicit WalletBase(State state) : SmartContract(std::move(state)) {
-  }
-  static td::Ref<WalletT> create(State state) {
-    return td::Ref<WalletT>(true, std::move(state));
-  }
-  static td::Ref<vm::Cell> get_init_code(int revision) {
-    return SmartContractCode::get_code(get_code_type(), revision);
-  };
-  size_t get_max_gifts_size() const override {
-    return Traits::max_gifts_size;
-  }
-  static SmartContractCode::Type get_code_type() {
-    return Traits::code_type;
-  }
-  static td::optional<td::int32> guess_revision(const vm::Cell::Hash& code_hash) {
-    for (auto i : ton::SmartContractCode::get_revisions(get_code_type())) {
-      auto code = SmartContractCode::get_code(get_code_type(), i);
-      if (code->get_hash() == code_hash) {
-        return i;
-      }
-    }
-    return {};
-  }
-
-  static td::Ref<WalletT> create(const InitData& init_data, int revision) {
-    return td::Ref<WalletT>(true, State{get_init_code(revision), WalletT::get_init_data(init_data)});
-  }
-
-  td::Result<td::uint32> get_seqno() const {
-    return TRY_VM([&]() -> td::Result<td::uint32> {
-      Answer answer = this->run_get_method("seqno");
-      if (!answer.success) {
-        return td::Status::Error("seqno get method failed");
-      }
-      return static_cast<td::uint32>(answer.stack.write().pop_long_range(std::numeric_limits<td::uint32>::max()));
-    }());
-  }
-  td::Result<td::uint32> get_wallet_id() const {
-    return TRY_VM([&]() -> td::Result<td::uint32> {
-      Answer answer = this->run_get_method("wallet_id");
-      if (!answer.success) {
-        return td::Status::Error("seqno get method failed");
-      }
-      return static_cast<td::uint32>(answer.stack.write().pop_long_range(std::numeric_limits<td::uint32>::max()));
-    }());
-  }
-
-  td::Result<td::uint64> get_balance(td::uint64 account_balance, td::uint32 now) const {
-    return TRY_VM([&]() -> td::Result<td::uint64> {
-      Answer answer = this->run_get_method(Args().set_method_id("balance").set_balance(account_balance).set_now(now));
-      if (!answer.success) {
-        return td::Status::Error("balance get method failed");
-      }
-      return static_cast<td::uint64>(answer.stack.write().pop_long());
-    }());
-  }
-
-  td::Result<td::Ed25519::PublicKey> get_public_key() const override {
-    return TRY_VM([&]() -> td::Result<td::Ed25519::PublicKey> {
-      Answer answer = this->run_get_method("get_public_key");
-      if (!answer.success) {
-        return td::Status::Error("get_public_key get method failed");
-      }
-      auto key_int = answer.stack.write().pop_int();
-      LOG(ERROR) << key_int->bit_size(false);
-      td::SecureString bytes(32);
-      if (!key_int->export_bytes(bytes.as_mutable_slice().ubegin(), bytes.size(), false)) {
-        return td::Status::Error("not a public key");
-      }
-      return td::Ed25519::PublicKey(std::move(bytes));
-    }());
-  };
-};
 
 struct RestrictedWalletTraits {
   struct InitData {

crypto/test/test-db.cpp

@@ -20,9 +20,9 @@
 #include "vm/cellslice.h"
 #include "vm/cells.h"
 #include "common/AtomicRef.h"
+#include "vm/cells/CellString.h"
 #include "vm/cells/MerkleProof.h"
 #include "vm/cells/MerkleUpdate.h"
-#include "vm/db/BlobView.h"
 #include "vm/db/CellStorage.h"
 #include "vm/db/CellHashTable.h"
 #include "vm/db/TonDb.h"
@@ -33,6 +33,7 @@
 #include "td/utils/crypto.h"
 #include "td/utils/Random.h"
 #include "td/utils/Slice.h"
+#include "td/utils/Span.h"
 #include "td/utils/Status.h"
 #include "td/utils/Timer.h"
 #include "td/utils/filesystem.h"
@@ -44,8 +45,12 @@
 #include "td/utils/tl_parsers.h"
 #include "td/utils/tl_helpers.h"
 
+#include "td/db/utils/BlobView.h"
 #include "td/db/RocksDb.h"
 #include "td/db/MemoryKeyValue.h"
+#include "td/db/utils/CyclicBuffer.h"
+
+#include "td/fec/fec.h"
 
 #include <set>
 #include <map>
@@ -433,17 +438,10 @@ class RandomBagOfCells {
   };
 };
 
-template <class T>
-void random_shuffle(td::MutableSpan<T> v, td::Random::Xorshift128plus &rnd) {
-  for (std::size_t i = 1; i < v.size(); i++) {
-    auto pos = static_cast<std::size_t>(rnd() % (i + 1));
-    std::swap(v[i], v[pos]);
-  }
-}
 Ref<Cell> gen_random_cell(int size, td::Random::Xorshift128plus &rnd, bool with_prunned_branches = true,
                           std::vector<Ref<Cell>> cells = {}) {
   if (!cells.empty()) {
-    random_shuffle(td::MutableSpan<Ref<Cell>>(cells), rnd);
+    td::random_shuffle(as_mutable_span(cells), rnd);
     cells.resize(cells.size() % rnd());
   }
   return RandomBagOfCells(size, rnd, with_prunned_branches, std::move(cells)).get_root();
@@ -451,7 +449,7 @@ Ref<Cell> gen_random_cell(int size, td::Random::Xorshift128plus &rnd, bool with_
 std::vector<Ref<Cell>> gen_random_cells(int roots, int size, td::Random::Xorshift128plus &rnd,
                                         bool with_prunned_branches = true, std::vector<Ref<Cell>> cells = {}) {
   if (!cells.empty()) {
-    random_shuffle(td::MutableSpan<Ref<Cell>>(cells), rnd);
+    td::random_shuffle(as_mutable_span(cells), rnd);
     cells.resize(cells.size() % rnd());
   }
   return RandomBagOfCells(size, rnd, with_prunned_branches, std::move(cells)).get_random_roots(roots, rnd);
@@ -788,7 +786,7 @@ TEST(TonDb, DynamicBoc) {
     old_root_serialization = serialize_boc(cell);
 
     // Check that DynamicBagOfCells properly loads cells
-    cell = vm::StaticBagOfCellsDbLazy::create(vm::BufferSliceBlobView::create(td::BufferSlice(old_root_serialization)))
+    cell = vm::StaticBagOfCellsDbLazy::create(td::BufferSliceBlobView::create(td::BufferSlice(old_root_serialization)))
                .move_as_ok()
                ->get_root_cell(0)
                .move_as_ok();
@@ -1599,11 +1597,11 @@ class BenchBocDeserializer : public td::Benchmark {
     auto blob = [&] {
       switch (config_.blob_type) {
         case BenchBocDeserializerConfig::File:
-          return vm::FileBlobView::create("serialization").move_as_ok();
+          return td::FileBlobView::create("serialization").move_as_ok();
         case BenchBocDeserializerConfig::Memory:
-          return vm::BufferSliceBlobView::create(serialization_.clone());
+          return td::BufferSliceBlobView::create(serialization_.clone());
         case BenchBocDeserializerConfig::FileMemoryMap:
-          return vm::FileMemoryMappingBlobView::create("serialization").move_as_ok();
+          return td::FileMemoryMappingBlobView::create("serialization").move_as_ok();
         default:
           UNREACHABLE();
       }
@@ -2083,222 +2081,6 @@ TEST(Ref, AtomicRef) {
   LOG(ERROR) << String::total_strings.sum();
 }
 
-class FileMerkleTree {
- public:
-  FileMerkleTree(size_t chunks_count, td::Ref<vm::Cell> root = {}) {
-    log_n_ = 0;
-    while ((size_t(1) << log_n_) < chunks_count) {
-      log_n_++;
-    }
-    n_ = size_t(1) << log_n_;
-    mark_.resize(n_ * 2);
-    proof_.resize(n_ * 2);
-
-    CHECK(n_ == chunks_count);  // TODO: support other chunks_count
-    //auto x = vm::CellBuilder().finalize();
-    root_ = std::move(root);
-  }
-
-  struct Chunk {
-    td::size_t index{0};
-    td::Slice hash;
-  };
-
-  void remove_chunk(td::size_t index) {
-    CHECK(index < n_);
-    index += n_;
-    while (proof_[index].not_null()) {
-      proof_[index] = {};
-      index /= 2;
-    }
-  }
-
-  bool has_chunk(td::size_t index) const {
-    CHECK(index < n_);
-    index += n_;
-    return proof_[index].not_null();
-  }
-
-  void add_chunk(td::size_t index, td::Slice hash) {
-    CHECK(hash.size() == 32);
-    CHECK(index < n_);
-    index += n_;
-    auto cell = vm::CellBuilder().store_bytes(hash).finalize();
-    CHECK(proof_[index].is_null());
-    proof_[index] = std::move(cell);
-    for (index /= 2; index != 0; index /= 2) {
-      CHECK(proof_[index].is_null());
-      auto &left = proof_[index * 2];
-      auto &right = proof_[index * 2 + 1];
-      if (left.not_null() && right.not_null()) {
-        proof_[index] = vm::CellBuilder().store_ref(left).store_ref(right).finalize();
-      } else {
-        mark_[index] = mark_id_;
-      }
-    }
-  }
-
-  td::Status validate_proof(td::Ref<vm::Cell> new_root) {
-    // TODO: check structure
-    return td::Status::OK();
-  }
-
-  td::Status add_proof(td::Ref<vm::Cell> new_root) {
-    TRY_STATUS(validate_proof(new_root));
-    auto combined = vm::MerkleProof::combine_fast_raw(root_, new_root);
-    if (combined.is_null()) {
-      return td::Status::Error("Can't combine proofs");
-    }
-    root_ = std::move(combined);
-    return td::Status::OK();
-  }
-
-  td::Status try_add_chunks(td::Span<Chunk> chunks) {
-    for (auto chunk : chunks) {
-      if (has_chunk(chunk.index)) {
-        return td::Status::Error("Already has chunk");
-      }
-    }
-    mark_id_++;
-    for (auto chunk : chunks) {
-      add_chunk(chunk.index, chunk.hash);
-    }
-    auto r_new_root = merge(root_, 1);
-    if (r_new_root.is_error()) {
-      for (auto chunk : chunks) {
-        remove_chunk(chunk.index);
-      }
-      return r_new_root.move_as_error();
-    }
-    root_ = r_new_root.move_as_ok();
-    return td::Status::OK();
-  }
-
-  td::Result<td::Ref<vm::Cell>> merge(td::Ref<vm::Cell> root, size_t index) {
-    const auto &down = proof_[index];
-    if (down.not_null()) {
-      if (down->get_hash() != root->get_hash(0)) {
-        return td::Status::Error("Hash mismatch");
-      }
-      return down;
-    }
-
-    if (mark_[index] != mark_id_) {
-      return root;
-    }
-
-    vm::CellSlice cs(vm::NoVm(), root);
-    if (cs.is_special()) {
-      return td::Status::Error("Proof is not enough to validate chunks");
-    }
-
-    CHECK(cs.size_refs() == 2);
-    vm::CellBuilder cb;
-    cb.store_bits(cs.fetch_bits(cs.size()));
-    TRY_RESULT(left, merge(cs.fetch_ref(), index * 2));
-    TRY_RESULT(right, merge(cs.fetch_ref(), index * 2 + 1));
-    cb.store_ref(std::move(left)).store_ref(std::move(right));
-    return cb.finalize();
-  }
-
-  void init_proof() {
-    CHECK(proof_[1].not_null());
-    root_ = proof_[1];
-  }
-
-  td::Result<td::Ref<vm::Cell>> gen_proof(size_t l, size_t r) {
-    auto usage_tree = std::make_shared<vm::CellUsageTree>();
-    auto usage_cell = vm::UsageCell::create(root_, usage_tree->root_ptr());
-    TRY_STATUS(do_gen_proof(std::move(usage_cell), 0, n_ - 1, l, r));
-    auto res = vm::MerkleProof::generate_raw(root_, usage_tree.get());
-    CHECK(res.not_null());
-    return res;
-  }
-
- private:
-  td::size_t n_;  // n = 2^log_n
-  td::size_t log_n_;
-  td::size_t mark_id_{0};
-  std::vector<td::size_t> mark_;          // n_ * 2
-  std::vector<td::Ref<vm::Cell>> proof_;  // n_ * 2
-  td::Ref<vm::Cell> root_;
-
-  td::Status do_gen_proof(td::Ref<vm::Cell> node, size_t il, size_t ir, size_t l, size_t r) {
-    if (ir < l || il > r) {
-      return td::Status::OK();
-    }
-    if (l <= il && ir <= r) {
-      return td::Status::OK();
-    }
-    vm::CellSlice cs(vm::NoVm(), std::move(node));
-    if (cs.is_special()) {
-      return td::Status::Error("Can't generate a proof");
-    }
-    CHECK(cs.size_refs() == 2);
-    auto ic = (il + ir) / 2;
-    TRY_STATUS(do_gen_proof(cs.fetch_ref(), il, ic, l, r));
-    TRY_STATUS(do_gen_proof(cs.fetch_ref(), ic + 1, ir, l, r));
-    return td::Status::OK();
-  }
-};
-
-TEST(FileMerkleTree, Manual) {
-  // create big random file
-  size_t chunk_size = 768;
-  // for simplicity numer of chunks in a file is a power of two
-  size_t chunks_count = 1 << 16;
-  size_t file_size = chunk_size * chunks_count;
-  td::Timer timer;
-  LOG(INFO) << "Generate random string";
-  const auto file = td::rand_string('a', 'z', td::narrow_cast<int>(file_size));
-  LOG(INFO) << timer;
-
-  timer = {};
-  LOG(INFO) << "Calculate all hashes";
-  std::vector<td::UInt256> hashes(chunks_count);
-  for (size_t i = 0; i < chunks_count; i++) {
-    td::sha256(td::Slice(file).substr(i * chunk_size, chunk_size), hashes[i].as_slice());
-  }
-  LOG(INFO) << timer;
-
-  timer = {};
-  LOG(INFO) << "Init merkle tree";
-  FileMerkleTree tree(chunks_count);
-  for (size_t i = 0; i < chunks_count; i++) {
-    tree.add_chunk(i, hashes[i].as_slice());
-  }
-  tree.init_proof();
-  LOG(INFO) << timer;
-
-  auto root_proof = tree.gen_proof(0, chunks_count - 1).move_as_ok();
-
-  // first download each chunk one by one
-
-  for (size_t stride : {1 << 6, 1}) {
-    timer = {};
-    LOG(INFO) << "Gen all proofs, stride = " << stride;
-    for (size_t i = 0; i < chunks_count; i += stride) {
-      tree.gen_proof(i, i + stride - 1).move_as_ok();
-    }
-    LOG(INFO) << timer;
-    timer = {};
-    LOG(INFO) << "Proof size: " << vm::std_boc_serialize(tree.gen_proof(0, stride - 1).move_as_ok()).ok().size();
-    LOG(INFO) << "Download file, stride = " << stride;
-    {
-      FileMerkleTree new_tree(chunks_count, root_proof);
-      for (size_t i = 0; i < chunks_count; i += stride) {
-        new_tree.add_proof(tree.gen_proof(i, i + stride - 1).move_as_ok()).ensure();
-        std::vector<FileMerkleTree::Chunk> chunks;
-        for (size_t j = 0; j < stride; j++) {
-          chunks.push_back({i + j, hashes[i + j].as_slice()});
-        }
-        new_tree.try_add_chunks(chunks).ensure();
-      }
-    }
-    LOG(INFO) << timer;
-  }
-}
-
 //TEST(Tmp, Boc) {
 //LOG(ERROR) << "A";
 //auto data = td::read_file("boc");

crypto/test/test-smartcont.cpp

@@ -34,9 +34,6 @@
 #include "smc-envelope/MultisigWallet.h"
 #include "smc-envelope/SmartContract.h"
 #include "smc-envelope/SmartContractCode.h"
-#include "smc-envelope/TestGiver.h"
-#include "smc-envelope/TestWallet.h"
-#include "smc-envelope/Wallet.h"
 #include "smc-envelope/WalletV3.h"
 #include "smc-envelope/HighloadWallet.h"
 #include "smc-envelope/HighloadWalletV2.h"
@@ -66,62 +63,6 @@ std::string load_source(std::string name) {
   return td::read_file_str(current_dir() + "../../crypto/" + name).move_as_ok();
 }
 
-td::Ref<vm::Cell> get_test_wallet_source() {
-  std::string code = R"ABCD(
-SETCP0 DUP IFNOTRET // return if recv_internal
-DUP 85143 INT EQUAL OVER 78748 INT EQUAL OR IFJMP:<{ // "seqno" and "get_public_key" get-methods
-  1 INT AND c4 PUSHCTR CTOS 32 LDU 256 PLDU CONDSEL  // cnt or pubk
-}>
-INC 32 THROWIF  // fail unless recv_external
-512 INT LDSLICEX DUP 32 PLDU   // sign cs cnt
-c4 PUSHCTR CTOS 32 LDU 256 LDU ENDS  // sign cs cnt cnt' pubk
-s1 s2 XCPU            // sign cs cnt pubk cnt' cnt
-EQUAL 33 THROWIFNOT   // ( seqno mismatch? )
-s2 PUSH HASHSU        // sign cs cnt pubk hash
-s0 s4 s4 XC2PU        // pubk cs cnt hash sign pubk
-CHKSIGNU              // pubk cs cnt ?
-34 THROWIFNOT         // signature mismatch
-ACCEPT
-SWAP 32 LDU NIP
-DUP SREFS IF:<{
-  // 3 INT 35 LSHIFT# 3 INT RAWRESERVE    // reserve all but 103 Grams from the balance
-  8 LDU LDREF         // pubk cnt mode msg cs
-  s0 s2 XCHG SENDRAWMSG  // pubk cnt cs ; ( message sent )
-}>
-ENDS
-INC NEWC 32 STU 256 STU ENDC c4 POPCTR
-)ABCD";
-  return fift::compile_asm(code).move_as_ok();
-}
-
-td::Ref<vm::Cell> get_wallet_source() {
-  std::string code = R"ABCD(
-SETCP0 DUP IFNOTRET // return if recv_internal
-   DUP 85143 INT EQUAL OVER 78748 INT EQUAL OR IFJMP:<{ // "seqno" and "get_public_key" get-methods
-     1 INT AND c4 PUSHCTR CTOS 32 LDU 256 PLDU CONDSEL  // cnt or pubk
-   }>
-   INC 32 THROWIF	// fail unless recv_external
-   9 PUSHPOW2 LDSLICEX DUP 32 LDU 32 LDU	//  signature in_msg msg_seqno valid_until cs
-   SWAP NOW LEQ 35 THROWIF	//  signature in_msg msg_seqno cs
-   c4 PUSH CTOS 32 LDU 256 LDU ENDS	//  signature in_msg msg_seqno cs stored_seqno public_key
-   s3 s1 XCPU	//  signature in_msg public_key cs stored_seqno msg_seqno stored_seqno
-   EQUAL 33 THROWIFNOT	//  signature in_msg public_key cs stored_seqno
-   s0 s3 XCHG HASHSU	//  signature stored_seqno public_key cs hash
-   s0 s4 s2 XC2PU CHKSIGNU 34 THROWIFNOT	//  cs stored_seqno public_key
-   ACCEPT
-   s0 s2 XCHG	//  public_key stored_seqno cs
-   WHILE:<{
-     DUP SREFS	//  public_key stored_seqno cs _40
-   }>DO<{	//  public_key stored_seqno cs
-     // 3 INT 35 LSHIFT# 3 INT RAWRESERVE    // reserve all but 103 Grams from the balance
-     8 LDU LDREF s0 s2 XCHG	//  public_key stored_seqno cs _45 mode
-     SENDRAWMSG	//  public_key stored_seqno cs
-   }>
-   ENDS INC	//  public_key seqno'
-   NEWC 32 STU 256 STU ENDC c4 POP
-)ABCD";
-  return fift::compile_asm(code).move_as_ok();
-}
 td::Ref<vm::Cell> get_wallet_v3_source() {
   std::string code = R"ABCD(
 SETCP0 DUP IFNOTRET // return if recv_internal
@@ -150,140 +91,37 @@ SETCP0 DUP IFNOTRET // return if recv_internal
   return fift::compile_asm(code).move_as_ok();
 }
 
-TEST(Tonlib, TestWallet) {
-  LOG(ERROR) << td::base64_encode(std_boc_serialize(get_test_wallet_source()).move_as_ok());
-  CHECK(get_test_wallet_source()->get_hash() == ton::TestWallet::get_init_code()->get_hash());
-  auto fift_output = fift::mem_run_fift(load_source("smartcont/new-wallet.fif"), {"aba", "0"}).move_as_ok();
-
-  auto new_wallet_pk = fift_output.source_lookup.read_file("new-wallet.pk").move_as_ok().data;
-  auto new_wallet_query = fift_output.source_lookup.read_file("new-wallet-query.boc").move_as_ok().data;
-  auto new_wallet_addr = fift_output.source_lookup.read_file("new-wallet.addr").move_as_ok().data;
-
-  td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}};
-  auto pub_key = priv_key.get_public_key().move_as_ok();
-  auto init_state = ton::TestWallet::get_init_state(pub_key);
-  auto init_message = ton::TestWallet::get_init_message_new(priv_key);
-  auto address = ton::GenericAccount::get_address(0, init_state);
-
-  CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32));
-
-  td::Ref<vm::Cell> res = ton::GenericAccount::create_ext_message(address, init_state, init_message);
-
-  LOG(ERROR) << "-------";
-  vm::load_cell_slice(res).print_rec(std::cerr);
-  LOG(ERROR) << "-------";
-  vm::load_cell_slice(vm::std_boc_deserialize(new_wallet_query).move_as_ok()).print_rec(std::cerr);
-  CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == res->get_hash());
-
-  fift_output.source_lookup.write_file("/main.fif", load_source("smartcont/wallet.fif")).ensure();
-  auto dest = block::StdAddress::parse("Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX").move_as_ok();
-  fift_output = fift::mem_run_fift(std::move(fift_output.source_lookup),
-                                   {"aba", "new-wallet", "Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX", "123",
-                                    "321", "-C", "TEST"})
-                    .move_as_ok();
-  auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data;
-  ton::TestWallet::Gift gift;
-  gift.destination = dest;
-  gift.message = "TEST";
-  gift.gramms = 321000000000ll;
-  ton::TestWallet wallet(priv_key.get_public_key().move_as_ok(), 123);
-  ASSERT_EQ(123u, wallet.get_seqno().ok());
-  CHECK(priv_key.get_public_key().ok().as_octet_string() == wallet.get_public_key().ok().as_octet_string());
-  auto gift_message = ton::GenericAccount::create_ext_message(
-      address, {}, wallet.make_a_gift_message(priv_key, 0, {gift}).move_as_ok());
-  LOG(ERROR) << "-------";
-  vm::load_cell_slice(gift_message).print_rec(std::cerr);
-  LOG(ERROR) << "-------";
-  vm::load_cell_slice(vm::std_boc_deserialize(wallet_query).move_as_ok()).print_rec(std::cerr);
-  CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == gift_message->get_hash());
-}
-
-td::Ref<vm::Cell> get_wallet_source_fc() {
-  return fift::compile_asm(load_source("smartcont/wallet-code.fif"), "", false).move_as_ok();
-}
-
-TEST(Tonlib, Wallet) {
-  LOG(ERROR) << td::base64_encode(std_boc_serialize(get_wallet_source()).move_as_ok());
-  CHECK(get_wallet_source()->get_hash() == ton::Wallet::get_init_code()->get_hash());
-
-  auto fift_output = fift::mem_run_fift(load_source("smartcont/new-wallet-v2.fif"), {"aba", "0"}).move_as_ok();
-
-  auto new_wallet_pk = fift_output.source_lookup.read_file("new-wallet.pk").move_as_ok().data;
-  auto new_wallet_query = fift_output.source_lookup.read_file("new-wallet-query.boc").move_as_ok().data;
-  auto new_wallet_addr = fift_output.source_lookup.read_file("new-wallet.addr").move_as_ok().data;
-
-  td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}};
-  auto pub_key = priv_key.get_public_key().move_as_ok();
-  auto init_state = ton::Wallet::get_init_state(pub_key);
-  auto init_message = ton::Wallet::get_init_message_new(priv_key);
-  auto address = ton::GenericAccount::get_address(0, init_state);
-
-  CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32));
-
-  td::Ref<vm::Cell> res = ton::GenericAccount::create_ext_message(address, init_state, init_message);
-
-  LOG(ERROR) << "-------";
-  vm::load_cell_slice(res).print_rec(std::cerr);
-  LOG(ERROR) << "-------";
-  vm::load_cell_slice(vm::std_boc_deserialize(new_wallet_query).move_as_ok()).print_rec(std::cerr);
-  CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == res->get_hash());
-
-  fift_output.source_lookup.write_file("/main.fif", load_source("smartcont/wallet-v2.fif")).ensure();
-  class ZeroOsTime : public fift::OsTime {
-   public:
-    td::uint32 now() override {
-      return 0;
-    }
-  };
-  fift_output.source_lookup.set_os_time(std::make_unique<ZeroOsTime>());
-  auto dest = block::StdAddress::parse("Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX").move_as_ok();
-  fift_output = fift::mem_run_fift(std::move(fift_output.source_lookup),
-                                   {"aba", "new-wallet", "-C", "TESTv2",
-                                    "Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX", "123", "321"})
-                    .move_as_ok();
-  auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data;
-  ton::TestWallet::Gift gift;
-  gift.destination = dest;
-  gift.message = "TESTv2";
-  gift.gramms = 321000000000ll;
-  ton::Wallet wallet(priv_key.get_public_key().move_as_ok(), 123);
-  ASSERT_EQ(123u, wallet.get_seqno().ok());
-  CHECK(priv_key.get_public_key().ok().as_octet_string() == wallet.get_public_key().ok().as_octet_string());
-  auto gift_message = ton::GenericAccount::create_ext_message(
-      address, {}, wallet.make_a_gift_message(priv_key, 60, {gift}).move_as_ok());
-  LOG(ERROR) << "-------";
-  vm::load_cell_slice(gift_message).print_rec(std::cerr);
-  LOG(ERROR) << "-------";
-  vm::load_cell_slice(vm::std_boc_deserialize(wallet_query).move_as_ok()).print_rec(std::cerr);
-  CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == gift_message->get_hash());
-}
-
 TEST(Tonlib, WalletV3) {
   LOG(ERROR) << td::base64_encode(std_boc_serialize(get_wallet_v3_source()).move_as_ok());
-  CHECK(get_wallet_v3_source()->get_hash() == ton::WalletV3::get_init_code()->get_hash());
+  CHECK(get_wallet_v3_source()->get_hash() == ton::WalletV3::get_init_code(2)->get_hash());
 
   auto fift_output = fift::mem_run_fift(load_source("smartcont/new-wallet-v3.fif"), {"aba", "0", "239"}).move_as_ok();
-
   auto new_wallet_pk = fift_output.source_lookup.read_file("new-wallet.pk").move_as_ok().data;
   auto new_wallet_query = fift_output.source_lookup.read_file("new-wallet-query.boc").move_as_ok().data;
   auto new_wallet_addr = fift_output.source_lookup.read_file("new-wallet.addr").move_as_ok().data;
 
   td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}};
   auto pub_key = priv_key.get_public_key().move_as_ok();
-  auto init_state = ton::WalletV3::get_init_state(pub_key, 239);
-  auto init_message =
-      ton::WalletV3(priv_key.get_public_key().move_as_ok(), 239).get_init_message(priv_key).move_as_ok();
-  auto address = ton::GenericAccount::get_address(0, init_state);
+  ton::WalletV3::InitData init_data;
+  init_data.public_key = pub_key.as_octet_string();
+  init_data.wallet_id = 239;
+  auto wallet = ton::WalletV3::create(init_data, 2);
+  ASSERT_EQ(239u, wallet->get_wallet_id().ok());
+  ASSERT_EQ(0u, wallet->get_seqno().ok());
 
+  auto address = wallet->get_address();
   CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32));
 
-  td::Ref<vm::Cell> res = ton::GenericAccount::create_ext_message(address, init_state, init_message);
-
+  auto init_message = wallet->get_init_message(priv_key).move_as_ok();
+  td::Ref<vm::Cell> ext_init_message = ton::GenericAccount::create_ext_message(
+      address, ton::GenericAccount::get_init_state(wallet->get_state()), init_message);
   LOG(ERROR) << "-------";
-  vm::load_cell_slice(res).print_rec(std::cerr);
+  vm::load_cell_slice(ext_init_message).print_rec(std::cerr);
   LOG(ERROR) << "-------";
   vm::load_cell_slice(vm::std_boc_deserialize(new_wallet_query).move_as_ok()).print_rec(std::cerr);
-  CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == res->get_hash());
+  CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == ext_init_message->get_hash());
+
+  CHECK(wallet.write().send_external_message(init_message).success);
 
   fift_output.source_lookup.write_file("/main.fif", load_source("smartcont/wallet-v3.fif")).ensure();
   class ZeroOsTime : public fift::OsTime {
@@ -296,7 +134,7 @@ TEST(Tonlib, WalletV3) {
   auto dest = block::StdAddress::parse("Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX").move_as_ok();
   fift_output = fift::mem_run_fift(std::move(fift_output.source_lookup),
                                    {"aba", "new-wallet", "-C", "TESTv3",
-                                    "Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX", "239", "123", "321"})
+                                    "Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX", "239", "1", "321"})
                     .move_as_ok();
   auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data;
 
@@ -305,15 +143,14 @@ TEST(Tonlib, WalletV3) {
   gift.message = "TESTv3";
   gift.gramms = 321000000000ll;
 
-  ton::WalletV3 wallet(priv_key.get_public_key().move_as_ok(), 239, 123);
-  ASSERT_EQ(239u, wallet.get_wallet_id().ok());
-  ASSERT_EQ(123u, wallet.get_seqno().ok());
-  CHECK(priv_key.get_public_key().ok().as_octet_string() == wallet.get_public_key().ok().as_octet_string());
+  ASSERT_EQ(239u, wallet->get_wallet_id().ok());
+  ASSERT_EQ(1u, wallet->get_seqno().ok());
+  CHECK(priv_key.get_public_key().ok().as_octet_string() == wallet->get_public_key().ok().as_octet_string());
   CHECK(priv_key.get_public_key().ok().as_octet_string() ==
-        ton::GenericAccount::get_public_key(wallet).ok().as_octet_string());
+        ton::GenericAccount::get_public_key(*wallet).ok().as_octet_string());
 
   auto gift_message = ton::GenericAccount::create_ext_message(
-      address, {}, wallet.make_a_gift_message(priv_key, 60, {gift}).move_as_ok());
+      address, {}, wallet->make_a_gift_message(priv_key, 60, {gift}).move_as_ok());
   LOG(ERROR) << "-------";
   vm::load_cell_slice(gift_message).print_rec(std::cerr);
   LOG(ERROR) << "-------";
@@ -334,20 +171,21 @@ TEST(Tonlib, HighloadWallet) {
 
   td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}};
   auto pub_key = priv_key.get_public_key().move_as_ok();
-  auto init_state = ton::HighloadWallet::get_init_state(pub_key, 239, -1);
-  auto init_message = ton::HighloadWallet::get_init_message(priv_key, 239);
-  auto address = ton::GenericAccount::get_address(0, init_state);
+  ton::HighloadWallet::InitData init_data(pub_key.as_octet_string(), 239);
 
-  ton::HighloadWallet wallet(
-      {ton::HighloadWallet::get_init_code(-1), ton::HighloadWallet::get_init_data(pub_key, 239)});
-  ASSERT_EQ(239u, wallet.get_wallet_id().ok());
-  ASSERT_EQ(0u, wallet.get_seqno().ok());
-  CHECK(pub_key.as_octet_string() == wallet.get_public_key().ok().as_octet_string());
-  CHECK(pub_key.as_octet_string() == ton::GenericAccount::get_public_key(wallet).ok().as_octet_string());
+  auto wallet = ton::HighloadWallet::create(init_data, -1);
+  auto address = wallet->get_address();
+  CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32));
+  ASSERT_EQ(239u, wallet->get_wallet_id().ok());
+  ASSERT_EQ(0u, wallet->get_seqno().ok());
+  CHECK(pub_key.as_octet_string() == wallet->get_public_key().ok().as_octet_string());
+  CHECK(pub_key.as_octet_string() == ton::GenericAccount::get_public_key(*wallet).ok().as_octet_string());
 
   CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32));
 
-  td::Ref<vm::Cell> res = ton::GenericAccount::create_ext_message(address, init_state, init_message);
+  auto init_message = wallet->get_init_message(priv_key).move_as_ok();
+  td::Ref<vm::Cell> res = ton::GenericAccount::create_ext_message(
+      address, ton::GenericAccount::get_init_state(wallet->get_state()), init_message);
 
   LOG(ERROR) << "---smc-envelope----";
   vm::load_cell_slice(res).print_rec(std::cerr);
@@ -382,12 +220,14 @@ TEST(Tonlib, HighloadWallet) {
       return 0;
     }
   };
+  init_data.seqno = 123;
+  wallet = ton::HighloadWallet::create(init_data, -1);
   fift_output.source_lookup.set_os_time(std::make_unique<ZeroOsTime>());
   fift_output = fift::mem_run_fift(std::move(fift_output.source_lookup), {"aba", "new-wallet", "239", "123", "order"})
                     .move_as_ok();
   auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data;
   auto gift_message = ton::GenericAccount::create_ext_message(
-      address, {}, ton::HighloadWallet::make_a_gift_message(priv_key, 239, 123, 60, gifts));
+      address, {}, wallet->make_a_gift_message(priv_key, 60, gifts).move_as_ok());
   LOG(ERROR) << "---smc-envelope----";
   vm::load_cell_slice(gift_message).print_rec(std::cerr);
   LOG(ERROR) << "---fift scripts----";
@@ -416,19 +256,21 @@ TEST(Tonlib, HighloadWalletV2) {
 
   td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}};
   auto pub_key = priv_key.get_public_key().move_as_ok();
-  auto init_state = ton::HighloadWalletV2::get_init_state(pub_key, 239, -1);
-  auto init_message = ton::HighloadWalletV2::get_init_message(priv_key, 239, 65535);
-  auto address = ton::GenericAccount::get_address(0, init_state);
+  ton::HighloadWalletV2::InitData init_data(pub_key.as_octet_string(), 239);
 
-  ton::HighloadWalletV2 wallet(
-      {ton::HighloadWalletV2::get_init_code(-1), ton::HighloadWalletV2::get_init_data(pub_key, 239)});
-  ASSERT_EQ(239u, wallet.get_wallet_id().ok());
-  CHECK(pub_key.as_octet_string() == wallet.get_public_key().ok().as_octet_string());
-  CHECK(pub_key.as_octet_string() == ton::GenericAccount::get_public_key(wallet).ok().as_octet_string());
+  auto wallet = ton::HighloadWalletV2::create(init_data, -1);
+  auto address = wallet->get_address();
+
+  ASSERT_EQ(239u, wallet->get_wallet_id().ok());
+  wallet->get_seqno().ensure_error();
+  CHECK(pub_key.as_octet_string() == wallet->get_public_key().ok().as_octet_string());
+  CHECK(pub_key.as_octet_string() == ton::GenericAccount::get_public_key(*wallet).ok().as_octet_string());
 
   CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32));
 
-  td::Ref<vm::Cell> res = ton::GenericAccount::create_ext_message(address, init_state, init_message);
+  auto init_message = wallet->get_init_message(priv_key, 65535).move_as_ok();
+  td::Ref<vm::Cell> res = ton::GenericAccount::create_ext_message(
+      address, ton::GenericAccount::get_init_state(wallet->get_state()), init_message);
 
   LOG(ERROR) << "---smc-envelope----";
   vm::load_cell_slice(res).print_rec(std::cerr);
@@ -462,7 +304,7 @@ TEST(Tonlib, HighloadWalletV2) {
       fift::mem_run_fift(std::move(fift_output.source_lookup), {"aba", "new-wallet", "239", "order"}).move_as_ok();
   auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data;
   auto gift_message = ton::GenericAccount::create_ext_message(
-      address, {}, ton::HighloadWalletV2::make_a_gift_message(priv_key, 239, 60, gifts));
+      address, {}, wallet->make_a_gift_message(priv_key, 60, gifts).move_as_ok());
   LOG(ERROR) << "---smc-envelope----";
   vm::load_cell_slice(gift_message).print_rec(std::cerr);
   LOG(ERROR) << "---fift scripts----";
@@ -470,28 +312,6 @@ TEST(Tonlib, HighloadWalletV2) {
   CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == gift_message->get_hash());
 }
 
-TEST(Tonlib, TestGiver) {
-  auto address =
-      block::StdAddress::parse("-1:60c04141c6a7b96d68615e7a91d265ad0f3a9a922e9ae9c901d4fa83f5d3c0d0").move_as_ok();
-  LOG(ERROR) << address.bounceable;
-  auto fift_output = fift::mem_run_fift(load_source("smartcont/testgiver.fif"),
-                                        {"aba", address.rserialize(), "0", "6.666", "wallet-query"})
-                         .move_as_ok();
-  LOG(ERROR) << fift_output.output;
-
-  auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data;
-
-  ton::TestGiver::Gift gift;
-  gift.gramms = 1000000000ll * 6666 / 1000;
-  gift.message = "GIFT";
-  gift.destination = address;
-  td::Ed25519::PrivateKey key{td::SecureString()};
-  auto res = ton::GenericAccount::create_ext_message(ton::TestGiver::address(), {},
-                                                     ton::TestGiver().make_a_gift_message(key, 0, {gift}).move_as_ok());
-  vm::CellSlice(vm::NoVm(), res).print_rec(std::cerr);
-  CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == res->get_hash());
-}
-
 TEST(Tonlib, RestrictedWallet) {
   //auto source_lookup = fift::create_mem_source_lookup(load_source("smartcont/new-restricted-wallet2.fif")).move_as_ok();
   //source_lookup
@@ -585,74 +405,130 @@ TEST(Tonlib, RestrictedWallet3) {
   CHECK(wallet->get_seqno().move_as_ok() == 2);
 }
 
-class SimpleWallet : public ton::SmartContract {
+template <class T>
+void check_wallet_seqno(td::Ref<T> wallet, td::uint32 seqno) {
+  ASSERT_EQ(seqno, wallet->get_seqno().ok());
+}
+void check_wallet_seqno(td::Ref<ton::HighloadWalletV2> wallet, td::uint32 seqno) {
+}
+void check_wallet_seqno(td::Ref<ton::WalletInterface> wallet, td::uint32 seqno) {
+}
+template <class T>
+void check_wallet_state(td::Ref<T> wallet, td::uint32 seqno, td::uint32 wallet_id, td::Slice public_key) {
+  ASSERT_EQ(wallet_id, wallet->get_wallet_id().ok());
+  ASSERT_EQ(public_key, wallet->get_public_key().ok().as_octet_string().as_slice());
+  check_wallet_seqno(wallet, seqno);
+}
+
+struct CreatedWallet {
+  td::optional<td::Ed25519::PrivateKey> priv_key;
+  block::StdAddress address;
+  td::Ref<ton::WalletInterface> wallet;
+};
+
+template <class T>
+class InitWallet {
  public:
-  SimpleWallet(State state) : SmartContract(std::move(state)) {
-  }
+  CreatedWallet operator()(int revision) const {
+    ton::WalletInterface::DefaultInitData init_data;
+    auto priv_key = td::Ed25519::generate_private_key().move_as_ok();
+    auto pub_key = priv_key.get_public_key().move_as_ok();
 
-  const State& get_state() const {
-    return state_;
-  }
-  SimpleWallet* make_copy() const override {
-    return new SimpleWallet{state_};
-  }
+    init_data.seqno = 0;
+    init_data.wallet_id = 123;
+    init_data.public_key = pub_key.as_octet_string();
 
-  static td::Ref<SimpleWallet> create_empty() {
-    return td::Ref<SimpleWallet>(true,
-                                 State{ton::SmartContractCode::get_code(ton::SmartContractCode::WalletV1Ext), {}});
-  }
-  static td::Ref<SimpleWallet> create(td::Ref<vm::Cell> data) {
-    return td::Ref<SimpleWallet>(
-        true, State{ton::SmartContractCode::get_code(ton::SmartContractCode::WalletV1Ext), std::move(data)});
-  }
-  static td::Ref<SimpleWallet> create_fast(td::Ref<vm::Cell> data) {
-    return td::Ref<SimpleWallet>(
-        true, State{ton::SmartContractCode::get_code(ton::SmartContractCode::WalletV1), std::move(data)});
-  }
+    auto wallet = T::create(init_data, revision);
+    auto address = wallet->get_address();
+    check_wallet_state(wallet, 0, 123, init_data.public_key);
+    CHECK(wallet.write().send_external_message(wallet->get_init_message(priv_key).move_as_ok()).success);
 
-  td::int32 seqno() const {
-    auto res = run_get_method("seqno");
-    return res.stack.write().pop_smallint_range(1000000000);
-  }
-
-  td::Ref<vm::Cell> create_init_state(td::Slice public_key) const {
-    td::RefInt256 pk{true};
-    pk.write().import_bytes(public_key.ubegin(), public_key.size(), false);
-    auto res = run_get_method("create_init_state", {pk});
-    return res.stack.write().pop_cell();
-  }
-
-  td::Ref<vm::Cell> prepare_send_message(td::Ref<vm::Cell> msg, td::int8 mode = 3) const {
-    auto res = run_get_method("prepare_send_message", {td::make_refint(mode), msg});
-    return res.stack.write().pop_cell();
-  }
-
-  static td::Ref<vm::Cell> sign_message(vm::Ref<vm::Cell> body, const td::Ed25519::PrivateKey& pk) {
-    auto signature = pk.sign(body->get_hash().as_slice()).move_as_ok();
-    return vm::CellBuilder().store_bytes(signature.as_slice()).append_cellslice(vm::load_cell_slice(body)).finalize();
+    CreatedWallet res;
+    res.wallet = std::move(wallet);
+    res.address = std::move(address);
+    res.priv_key = std::move(priv_key);
+    return res;
   }
 };
 
-TEST(Smartcon, Simple) {
-  auto private_key = td::Ed25519::generate_private_key().move_as_ok();
-  auto public_key = private_key.get_public_key().move_as_ok().as_octet_string();
+template <>
+CreatedWallet InitWallet<ton::RestrictedWallet>::operator()(int revision) const {
+  auto init_priv_key = td::Ed25519::generate_private_key().move_as_ok();
+  auto init_pub_key = init_priv_key.get_public_key().move_as_ok();
+  auto priv_key = td::Ed25519::generate_private_key().move_as_ok();
+  auto pub_key = priv_key.get_public_key().move_as_ok();
 
-  auto w_lib = SimpleWallet::create_empty();
-  auto init_data = w_lib->create_init_state(public_key);
+  ton::RestrictedWallet::InitData init_data;
+  init_data.init_key = init_pub_key.as_octet_string();
+  init_data.main_key = pub_key.as_octet_string();
+  init_data.wallet_id = 123;
+  auto wallet = ton::RestrictedWallet::create(init_data, 1);
+  check_wallet_state(wallet, 0, 123, init_data.init_key);
 
-  auto w = SimpleWallet::create(init_data);
-  LOG(ERROR) << w->code_size();
-  auto fw = SimpleWallet::create_fast(init_data);
-  LOG(ERROR) << fw->code_size();
-  LOG(ERROR) << w->seqno();
+  auto address = wallet->get_address();
 
-  for (int i = 0; i < 20; i++) {
-    auto msg = w->sign_message(w->prepare_send_message(vm::CellBuilder().finalize()), private_key);
-    w.write().send_external_message(msg);
-    fw.write().send_external_message(msg);
+  td::uint64 x = 100 * 1000000000ull;
+  ton::RestrictedWallet::Config config;
+  config.start_at = 1;
+  config.limits = {{-32768, x}, {92, x * 3 / 4}, {183, x * 1 / 2}, {366, x * 1 / 4}, {548, 0}};
+  CHECK(wallet.write().send_external_message(wallet->get_init_message(init_priv_key, 10, config).move_as_ok()).success);
+  CHECK(wallet->get_seqno().move_as_ok() == 1);
+
+  CreatedWallet res;
+  res.wallet = std::move(wallet);
+  res.address = std::move(address);
+  res.priv_key = std::move(priv_key);
+  return res;
+}
+
+template <class T>
+void do_test_wallet(int revision) {
+  auto res = InitWallet<T>()(revision);
+  auto priv_key = res.priv_key.unwrap();
+  auto address = std::move(res.address);
+  auto iwallet = std::move(res.wallet);
+  auto public_key = priv_key.get_public_key().move_as_ok().as_octet_string();
+  ;
+  check_wallet_state(iwallet, 1, 123, public_key);
+
+  // lets send a lot of messages
+  std::vector<ton::WalletInterface::Gift> gifts;
+  for (size_t i = 0; i < iwallet->get_max_gifts_size(); i++) {
+    ton::WalletInterface::Gift gift;
+    gift.gramms = 1;
+    gift.destination = address;
+    gift.message = std::string(iwallet->get_max_message_size(), 'z');
+    gifts.push_back(gift);
   }
-  ASSERT_EQ(20, w->seqno());
-  CHECK(w->get_state().data->get_hash() == fw->get_state().data->get_hash());
+
+  td::uint32 valid_until = 10000;
+  auto send_gifts = iwallet->make_a_gift_message(priv_key, valid_until, gifts).move_as_ok();
+
+  {
+    auto cwallet = iwallet;
+    CHECK(!cwallet.write()
+               .send_external_message(send_gifts, ton::SmartContract::Args().set_now(valid_until + 1))
+               .success);
+  }
+  //TODO: make wallet work (or not) with now == valid_until
+  auto ans = iwallet.write().send_external_message(send_gifts, ton::SmartContract::Args().set_now(valid_until - 1));
+  CHECK(ans.success);
+  CHECK((int)gifts.size() <= ans.output_actions_count(ans.actions));
+  check_wallet_state(iwallet, 2, 123, public_key);
+}
+
+template <class T>
+void do_test_wallet() {
+  for (auto revision : T::get_revisions()) {
+    do_test_wallet<T>(revision);
+  }
+}
+
+TEST(Tonlib, Wallet) {
+  do_test_wallet<ton::WalletV3>();
+  do_test_wallet<ton::HighloadWallet>();
+  do_test_wallet<ton::HighloadWalletV2>();
+  do_test_wallet<ton::RestrictedWallet>();
 }
 
 namespace std {  // ouch
@@ -1157,7 +1033,7 @@ class CheckedDns {
     }
   }
   void update(const Action& action) {
-    return update(td::Span<Action>(&action, 1));
+    return update(td::span_one(action));
   }
 
   std::vector<Entry> resolve(td::Slice name, td::int16 category) {
@@ -1337,16 +1213,13 @@ TEST(Smartcont, DnsManual) {
   {
     CheckedDns::Action e[4] = {CheckedDns::Action{"", 0, ""}, CheckedDns::Action{"a.b.c", 1, "hello"},
                                CheckedDns::Action{"a.b.c", 2, "world"}, CheckedDns::Action{"x.y.z", 3, "abc"}};
-    dns.update(td::Span<CheckedDns::Action>(e, 4));
+    dns.update(td::span(e, 4));
   }
   dns.resolve("a.b.c", 1);
   dns.resolve("a.b.c", 2);
   dns.resolve("x.y.z", 3);
 
-  {
-    CheckedDns::Action e[1] = {CheckedDns::Action{"x.y.z", 0, ""}};
-    dns.update(td::Span<CheckedDns::Action>(e, 1));
-  }
+  dns.update(td::span_one(CheckedDns::Action{"x.y.z", 0, ""}));
 
   dns.resolve("a.b.c", 1);
   dns.resolve("a.b.c", 2);
@@ -1355,7 +1228,7 @@ TEST(Smartcont, DnsManual) {
   {
     CheckedDns::Action e[3] = {CheckedDns::Action{"x.y.z", 0, ""}, CheckedDns::Action{"x.y.z", 1, "xxx"},
                                CheckedDns::Action{"x.y.z", 2, "yyy"}};
-    dns.update(td::Span<CheckedDns::Action>(e, 3));
+    dns.update(td::span(e, 3));
   }
   dns.resolve("a.b.c", 1);
   dns.resolve("a.b.c", 2);

crypto/vm/cells/CellSlice.cpp

@@ -48,7 +48,7 @@ Cell::LoadedCell load_cell_nothrow(const Ref<Cell>& ref) {
   auto res = ref->load_cell();
   if (res.is_ok()) {
     auto ld = res.move_as_ok();
-    CHECK(ld.virt.get_virtualization() == 0 || ld.data_cell->special_type() != Cell::SpecialType::PrunnedBranch);
+    //CHECK(ld.virt.get_virtualization() == 0 || ld.data_cell->special_type() != Cell::SpecialType::PrunnedBranch);
     return ld;
   }
   return {};
@@ -58,7 +58,7 @@ Cell::LoadedCell load_cell_nothrow(const Ref<Cell>& ref, int mode) {
   auto res = ref->load_cell();
   if (res.is_ok()) {
     auto ld = res.move_as_ok();
-    CHECK(ld.virt.get_virtualization() == 0 || ld.data_cell->special_type() != Cell::SpecialType::PrunnedBranch);
+    //CHECK(ld.virt.get_virtualization() == 0 || ld.data_cell->special_type() != Cell::SpecialType::PrunnedBranch);
     if ((mode >> (ld.data_cell->is_special() ? 1 : 0)) & 1) {
       return ld;
     }

crypto/vm/cells/CellString.cpp

@@ -79,6 +79,19 @@ td::Result<td::string> CellString::load(CellSlice &cs, unsigned int top_bits) {
   CHECK(to.offs == (int)size);
   return res;
 }
+td::Result<td::Ref<vm::Cell>> CellString::create(td::Slice slice, unsigned int top_bits) {
+  vm::CellBuilder cb;
+  TRY_STATUS(store(cb, slice, top_bits));
+  return cb.finalize();
+}
+bool CellString::fetch_to(CellSlice &cs, std::string &res, unsigned int top_bits) {
+  auto r_str = load(cs, top_bits);
+  if (r_str.is_error()) {
+    return false;
+  }
+  res = r_str.move_as_ok();
+  return true;
+}
 
 td::Status CellText::store(CellBuilder &cb, td::Slice slice, unsigned int top_bits) {
   td::uint32 size = td::narrow_cast<td::uint32>(slice.size() * 8);
@@ -154,4 +167,17 @@ td::Result<td::string> CellText::load(CellSlice &cs) {
   CHECK(to.offs == (int)size);
   return res;
 }
+td::Result<td::Ref<vm::Cell>> CellText::create(td::Slice slice, unsigned int top_bits) {
+  vm::CellBuilder cb;
+  TRY_STATUS(store(cb, slice, top_bits));
+  return cb.finalize();
+}
+bool CellText::fetch_to(CellSlice &cs, std::string &res) {
+  auto r_str = load(cs);
+  if (r_str.is_error()) {
+    return false;
+  }
+  res = r_str.move_as_ok();
+  return true;
+}
 }  // namespace vm

crypto/vm/cells/CellString.h

@@ -31,11 +31,8 @@ class CellString {
   static td::Status store(CellBuilder &cb, td::Slice slice, unsigned int top_bits = Cell::max_bits);
   static td::Status store(CellBuilder &cb, td::BitSlice slice, unsigned int top_bits = Cell::max_bits);
   static td::Result<td::string> load(CellSlice &cs, unsigned int top_bits = Cell::max_bits);
-  static td::Result<td::Ref<vm::Cell>> create(td::Slice slice, unsigned int top_bits = Cell::max_bits) {
-    vm::CellBuilder cb;
-    TRY_STATUS(store(cb, slice, top_bits));
-    return cb.finalize();
-  }
+  static td::Result<td::Ref<vm::Cell>> create(td::Slice slice, unsigned int top_bits = Cell::max_bits);
+  static bool fetch_to(CellSlice &cs, std::string &res, unsigned int top_bits = Cell::max_bits);
 
  private:
   template <class F>
@@ -50,11 +47,8 @@ class CellText {
   static td::Status store(CellBuilder &cb, td::Slice slice, unsigned int top_bits = Cell::max_bits);
   static td::Status store(CellBuilder &cb, td::BitSlice slice, unsigned int top_bits = Cell::max_bits);
   static td::Result<td::string> load(CellSlice &cs);
-  static td::Result<td::Ref<vm::Cell>> create(td::Slice slice, unsigned int top_bits = Cell::max_bits) {
-    vm::CellBuilder cb;
-    TRY_STATUS(store(cb, slice, top_bits));
-    return cb.finalize();
-  }
+  static td::Result<td::Ref<vm::Cell>> create(td::Slice slice, unsigned int top_bits = Cell::max_bits);
+  static bool fetch_to(CellSlice &cs, std::string &res);
 
  private:
   template <class F>

crypto/vm/db/StaticBagOfCellsDb.cpp

@@ -187,9 +187,9 @@ class StaticBagOfCellsDbBaselineImpl : public StaticBagOfCellsDb {
   }
 };
 
-td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbBaseline::create(std::unique_ptr<BlobView> data) {
-  std::string buf(data->size(), '\0');
-  TRY_RESULT(slice, data->view(buf, 0));
+td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbBaseline::create(td::BlobView data) {
+  std::string buf(data.size(), '\0');
+  TRY_RESULT(slice, data.view(buf, 0));
   return create(slice);
 }
 
@@ -211,7 +211,7 @@ td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbBaseline::crea
 //
 class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb {
  public:
-  explicit StaticBagOfCellsDbLazyImpl(std::unique_ptr<BlobView> data, StaticBagOfCellsDbLazy::Options options)
+  explicit StaticBagOfCellsDbLazyImpl(td::BlobView data, StaticBagOfCellsDbLazy::Options options)
       : data_(std::move(data)), options_(std::move(options)) {
     get_thread_safe_counter().add(1);
   }
@@ -240,7 +240,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb {
 
  private:
   std::atomic<bool> should_cache_cells_{true};
-  std::unique_ptr<BlobView> data_;
+  td::BlobView data_;
   StaticBagOfCellsDbLazy::Options options_;
   bool has_info_{false};
   BagOfCells::Info info_;
@@ -313,8 +313,8 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb {
     char arr[8];
     td::RwMutex::ReadLock guard;
     if (info_.has_index) {
-      TRY_RESULT(new_offset_view, data_->view(td::MutableSlice(arr, info_.offset_byte_size),
-                                              info_.index_offset + idx * info_.offset_byte_size));
+      TRY_RESULT(new_offset_view, data_.view(td::MutableSlice(arr, info_.offset_byte_size),
+                                             info_.index_offset + idx * info_.offset_byte_size));
       offset_view = new_offset_view;
     } else {
       guard = index_data_rw_mutex_.lock_read().move_as_ok();
@@ -331,8 +331,8 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb {
       return 0;
     }
     char arr[8];
-    TRY_RESULT(idx_view, data_->view(td::MutableSlice(arr, info_.ref_byte_size),
-                                     info_.roots_offset + root_i * info_.ref_byte_size));
+    TRY_RESULT(idx_view, data_.view(td::MutableSlice(arr, info_.ref_byte_size),
+                                    info_.roots_offset + root_i * info_.ref_byte_size));
     CHECK(idx_view.size() == (size_t)info_.ref_byte_size);
     return info_.read_ref(idx_view.ubegin());
   }
@@ -369,17 +369,17 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb {
       return td::Status::OK();
     }
     std::string header(1000, '\0');
-    TRY_RESULT(header_view, data_->view(td::MutableSlice(header).truncate(data_->size()), 0))
+    TRY_RESULT(header_view, data_.view(td::MutableSlice(header).truncate(data_.size()), 0))
     auto parse_res = info_.parse_serialized_header(header_view);
     if (parse_res <= 0) {
       return td::Status::Error("bag-of-cell error: failed to read header");
     }
-    if (info_.total_size < data_->size()) {
+    if (info_.total_size < data_.size()) {
       return td::Status::Error("bag-of-cell error: not enough data");
     }
     if (options_.check_crc32c && info_.has_crc32c) {
       std::string buf(td::narrow_cast<std::size_t>(info_.total_size), '\0');
-      TRY_RESULT(data, data_->view(td::MutableSlice(buf), 0));
+      TRY_RESULT(data, data_.view(td::MutableSlice(buf), 0));
       unsigned crc_computed = td::crc32c(td::Slice{data.ubegin(), data.uend() - 4});
       unsigned crc_stored = td::as<unsigned>(data.uend() - 4);
       if (crc_computed != crc_stored) {
@@ -407,8 +407,8 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb {
     auto buf_slice = td::MutableSlice(buf.data(), buf.size());
     for (; index_i_ <= idx; index_i_++) {
       auto offset = td::narrow_cast<size_t>(info_.data_offset + index_offset_);
-      CHECK(data_->size() >= offset);
-      TRY_RESULT(cell, data_->view(buf_slice.copy().truncate(data_->size() - offset), offset));
+      CHECK(data_.size() >= offset);
+      TRY_RESULT(cell, data_.view(buf_slice.copy().truncate(data_.size() - offset), offset));
       CellSerializationInfo cell_info;
       TRY_STATUS(cell_info.init(cell, info_.ref_byte_size));
       index_offset_ += cell_info.end_offset;
@@ -455,7 +455,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb {
 
     TRY_RESULT(cell_location, get_cell_location(idx));
     auto buf = alloc(cell_location.end - cell_location.begin);
-    TRY_RESULT(cell_slice, data_->view(buf.as_slice(), cell_location.begin));
+    TRY_RESULT(cell_slice, data_.view(buf.as_slice(), cell_location.begin));
     TRY_RESULT(res, deserialize_any_cell(idx, cell_slice, cell_location.should_cache));
     return std::move(res);
   }
@@ -470,7 +470,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb {
 
     TRY_RESULT(cell_location, get_cell_location(idx));
     auto buf = alloc(cell_location.end - cell_location.begin);
-    TRY_RESULT(cell_slice, data_->view(buf.as_slice(), cell_location.begin));
+    TRY_RESULT(cell_slice, data_.view(buf.as_slice(), cell_location.begin));
     TRY_RESULT(res, deserialize_data_cell(idx, cell_slice, cell_location.should_cache));
     return std::move(res);
   }
@@ -528,18 +528,17 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb {
   }
 };
 
-td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbLazy::create(std::unique_ptr<BlobView> data,
-                                                                               Options options) {
+td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbLazy::create(td::BlobView data, Options options) {
   return std::make_shared<StaticBagOfCellsDbLazyImpl>(std::move(data), std::move(options));
 }
 
 td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbLazy::create(td::BufferSlice data, Options options) {
-  return std::make_shared<StaticBagOfCellsDbLazyImpl>(vm::BufferSliceBlobView::create(std::move(data)),
+  return std::make_shared<StaticBagOfCellsDbLazyImpl>(td::BufferSliceBlobView::create(std::move(data)),
                                                       std::move(options));
 }
 
 td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbLazy::create(std::string data, Options options) {
-  return create(BufferSliceBlobView::create(td::BufferSlice(data)), std::move(options));
+  return create(td::BufferSliceBlobView::create(td::BufferSlice(data)), std::move(options));
 }
 
 }  // namespace vm

crypto/vm/db/StaticBagOfCellsDb.h

@@ -19,7 +19,7 @@
 #pragma once
 
 #include "vm/cells.h"
-#include "vm/db/BlobView.h"
+#include "td/db/utils/BlobView.h"
 
 #include "td/utils/Status.h"
 
@@ -41,7 +41,7 @@ class StaticBagOfCellsDb : public std::enable_shared_from_this<StaticBagOfCellsD
 
 class StaticBagOfCellsDbBaseline {
  public:
-  static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(std::unique_ptr<BlobView> data);
+  static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(td::BlobView data);
   static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(td::Slice data);
 };
 
@@ -52,7 +52,7 @@ class StaticBagOfCellsDbLazy {
     }
     bool check_crc32c{false};
   };
-  static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(std::unique_ptr<BlobView> data, Options options = {});
+  static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(td::BlobView data, Options options = {});
   static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(td::BufferSlice data, Options options = {});
   static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(std::string data, Options options = {});
 };

lite-client/lite-client.cpp

@@ -1129,7 +1129,7 @@ td::Result<std::pair<Ref<vm::Cell>, std::shared_ptr<vm::StaticBagOfCellsDb>>> la
     td::BufferSlice data) {
   vm::StaticBagOfCellsDbLazy::Options options;
   options.check_crc32c = true;
-  TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(vm::BufferSliceBlobView::create(std::move(data)), options));
+  TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(td::BufferSliceBlobView::create(std::move(data)), options));
   TRY_RESULT(rc, boc->get_root_count());
   if (rc != 1) {
     return td::Status::Error(-668, "bag-of-cells is not standard (exactly one root cell expected)");

rldp2/Ack.cpp

@@ -0,0 +1,29 @@
+#include "Ack.h"
+
+namespace ton {
+namespace rldp2 {
+
+bool Ack::on_got_packet(td::uint32 seqno) {
+  if (seqno > max_seqno) {
+    td::uint32 diff = seqno - max_seqno;
+    if (diff >= 32) {
+      received_mask = 0;
+    } else {
+      received_mask <<= diff;
+    }
+    max_seqno = seqno;
+  }
+  td::uint32 offset = max_seqno - seqno;
+  if (offset < 32) {
+    td::uint32 mask = 1 << offset;
+    if ((received_mask & mask) == 0) {
+      received_count++;
+      received_mask |= mask;
+      return true;
+    }
+  }
+
+  return false;
+}
+}  // namespace rldp2
+}  // namespace ton

rldp2/Ack.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "td/utils/int_types.h"
+
+namespace ton {
+namespace rldp2 {
+// Helper for receiver
+// Also this information is sent to the sender as an acknowlegement.
+struct Ack {
+  td::uint32 max_seqno{0};
+  td::uint32 received_mask{0};
+  td::uint32 received_count{0};
+
+  // returns true if we know that packet is new and hasn't been received yet
+  bool on_got_packet(td::uint32 seqno);
+};
+}  // namespace rldp2
+}  // namespace ton

rldp2/Bbr.cpp

@@ -0,0 +1,62 @@
+#include "Bbr.h"
+
+#include "BdwStats.h"
+#include "RttStats.h"
+
+#include "td/utils/Random.h"
+
+namespace ton {
+namespace rldp2 {
+void Bbr::step(const RttStats &rtt_stats, const BdwStats &bdw_stats, td::uint64 in_flight, td::Timestamp now) {
+  rtt_min_ = rtt_stats.windowed_min_rtt;
+  bdw_max_ = bdw_stats.windowed_max_bdw;
+  if (bdw_max_ > bdw_peak_ * 1.25) {
+    bdw_peak_ = bdw_max_;
+    bdw_peak_at_round = rtt_stats.rtt_round;
+    //LOG(ERROR) << "NEW PEAK " << bdw_peak_ * 768;
+  }
+
+  if (state_ == State::Start && bdw_peak_at_round + 3 < rtt_stats.rtt_round) {
+    //LOG(ERROR) << "START -> DRAIN";
+    state_ = State::Drain;
+  }
+
+  if (state_ == State::Drain && (double)in_flight < bdw_max_ * rtt_min_) {
+    //LOG(ERROR) << "DRAIN -> BPROBE BDW";
+    state_ = State::ProbeBdw;
+    probe_bdw_cycle_ = td::Random::fast(1, 5);
+    probe_bdw_cycle_at_ = now;
+  }
+
+  if (state_ == State::ProbeBdw && td::Timestamp::in(rtt_stats.windowed_min_rtt, probe_bdw_cycle_at_).is_in_past(now)) {
+    probe_bdw_cycle_at_ = now;
+    probe_bdw_cycle_ = (probe_bdw_cycle_ + 1) % 6;
+    //LOG(ERROR) << "NEW PROBE BDW CYCLE";
+  }
+
+  //TODO: ProbeRtt state. Don't want to implenent now without proper tests
+}
+
+double Bbr::get_rate() const {
+  if (state_ == State::Start) {
+    return bdw_max_ * 2.8;
+  }
+  if (state_ == State::Drain) {
+    return bdw_max_ / 2.8;
+  }
+  if (state_ == State::ProbeBdw) {
+    constexpr double probe_bdw_gain[6] = {0.75, 1, 1, 1, 1, 1.25};
+    return probe_bdw_gain[probe_bdw_cycle_] * bdw_max_;
+  }
+  UNREACHABLE();
+}
+
+td::uint32 Bbr::get_window_size() const {
+  if (state_ == State::Start || state_ == State::Drain) {
+    return td::max(td::uint32(bdw_max_ * rtt_min_ * 2.8 + 1), 10u);
+  }
+  return td::max(td::uint32(bdw_max_ * rtt_min_ * 2 + 1), 10u);
+}
+
+}  // namespace rldp2
+}  // namespace ton

rldp2/Bbr.h

@@ -0,0 +1,29 @@
+#pragma once
+#include "td/utils/int_types.h"
+#include "td/utils/Time.h"
+
+namespace ton {
+namespace rldp2 {
+struct RttStats;
+struct BdwStats;
+
+struct Bbr {
+ public:
+  void step(const RttStats &rtt_stats, const BdwStats &bdw_stats, td::uint64 in_flight, td::Timestamp now);
+  double get_rate() const;
+
+  td::uint32 get_window_size() const;
+
+ private:
+  double bdw_peak_{-1};
+  td::uint32 bdw_peak_at_round{0};
+  td::uint32 probe_bdw_cycle_{0};
+  td::Timestamp probe_bdw_cycle_at_;
+  double rtt_min_{0};
+  double bdw_max_{0};
+  enum class State { Start, Drain, ProbeRtt, ProbeBdw } state_ = State::Start;
+};
+
+}  // namespace rldp2
+}  // namespace ton
+

rldp2/BdwStats.cpp

@@ -0,0 +1,50 @@
+#include "BdwStats.h"
+
+namespace ton {
+namespace rldp2 {
+
+BdwStats::PacketInfo BdwStats::on_packet_send(td::Timestamp first_sent_at) const {
+  PacketInfo packet;
+  packet.delivered_now = delivered_now;
+  packet.first_sent_at = first_sent_at;
+  packet.delivered_count = delivered_count;
+  packet.is_paused = static_cast<bool>(paused_at_);
+  return packet;
+}
+
+void BdwStats::on_packet_ack(const PacketInfo &info, td::Timestamp sent_at, td::Timestamp now) {
+  if (paused_at_.is_in_past(info.delivered_now)) {
+    paused_at_ = {};
+  }
+  auto sent_passed = sent_at.at() - info.first_sent_at.at();
+  auto ack_passed = now.at() - info.delivered_now.at();
+  auto passed = td::max(sent_passed, ack_passed);
+  if (passed < 0.01) {
+    LOG(ERROR) << "Invalid passed " << passed;
+  }
+  auto delivered = delivered_count - info.delivered_count;
+  on_rate_sample((double)delivered / passed, now, info.is_paused);
+}
+
+void BdwStats::on_update(td::Timestamp now, td::uint64 delivered_count_diff) {
+  this->delivered_now = now;
+  this->delivered_count += delivered_count_diff;
+}
+
+void BdwStats::on_pause(td::Timestamp now) {
+  paused_at_ = now;
+}
+
+void BdwStats::on_rate_sample(double rate, td::Timestamp now, bool is_paused) {
+  // ignore decrease of rate if is_paused == true
+  if (is_paused && rate < windowed_max_bdw) {
+    return;
+  }
+  windowed_max_bdw_stat.add_event(rate, now.at());
+  auto windowed_max_bdw_sample = windowed_max_bdw_stat.get_stat(now.at()).get_stat();
+  if (windowed_max_bdw_sample) {
+    windowed_max_bdw = windowed_max_bdw_sample.value();
+  }
+}
+}  // namespace rldp2
+}  // namespace ton

rldp2/BdwStats.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include "td/utils/Time.h"
+#include "td/utils/TimedStat.h"
+
+namespace ton {
+namespace rldp2 {
+struct BdwStats {
+  struct State {};
+
+  struct PacketInfo {
+    td::Timestamp first_sent_at;
+
+    td::Timestamp delivered_now;
+    td::uint64 delivered_count{0};
+    bool is_paused{false};
+  };
+
+  PacketInfo on_packet_send(td::Timestamp first_sent_at) const;
+  void on_packet_ack(const PacketInfo &info, td::Timestamp sent_at, td::Timestamp now);
+
+  void on_update(td::Timestamp now, td::uint64 delivered_count_diff);
+
+  void on_pause(td::Timestamp now);
+  double windowed_max_bdw{0};
+
+ private:
+  td::Timestamp delivered_now;
+  td::uint64 delivered_count{0};
+  td::TimedStat<td::MaxStat<double>> windowed_max_bdw_stat{5, 0};
+  td::Timestamp paused_at_;
+
+  void on_rate_sample(double rate, td::Timestamp now, bool is_paused);
+};
+}  // namespace rldp2
+}  // namespace ton

rldp2/CMakeLists.txt

@@ -0,0 +1,58 @@
+cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
+
+if (NOT OPENSSL_FOUND)
+  find_package(OpenSSL REQUIRED)
+endif()
+
+if (NOT GSL_FOUND)
+  find_package(GSL)
+endif()
+
+set(RLDP_SOURCE
+  Ack.cpp
+  Bbr.cpp
+  BdwStats.cpp
+  FecHelper.cpp
+  InboundTransfer.cpp
+  LossSender.cpp
+  LossStats.cpp
+  OutboundTransfer.cpp
+  Pacer.cpp
+  rldp.cpp
+  RldpReceiver.cpp
+  RldpSender.cpp
+  RldpConnection.cpp
+  RttStats.cpp
+  SenderPackets.cpp
+
+  Ack.h
+  Bbr.h
+  BdwStats.h
+  FecHelper.h
+  InboundTransfer.h
+  LossSender.h
+  LossStats.h
+  OutboundTransfer.h
+  Pacer.h
+  rldp.h
+  rldp.hpp
+  RldpReceiver.h
+  RldpSender.h
+  RldpConnection.h
+  RttStats.h
+  SenderPackets.h
+)
+
+add_library(rldp2 STATIC ${RLDP_SOURCE})
+
+target_include_directories(rldp PUBLIC 
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}> 
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/.. 
+  ${OPENSSL_INCLUDE_DIR}
+)
+if (GSL_FOUND)
+  target_link_libraries(rldp2 PRIVATE GSL::gsl)
+  target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1)
+endif()
+target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api)
+

rldp2/FecHelper.cpp

@@ -0,0 +1,24 @@
+#include "FecHelper.h"
+
+#include "td/utils/check.h"
+
+namespace ton {
+namespace rldp2 {
+td::uint32 FecHelper::get_fec_symbols_count() const {
+  constexpr td::uint32 x = 5;
+  constexpr td::uint32 y = 5;
+  // smallest (symbols_count + x + y * i) >  received_symbols_count
+  if (symbols_count + x > received_symbols_count) {
+    return symbols_count + x;
+  }
+  td::uint32 i = (received_symbols_count - (symbols_count + x)) / y + 1;
+  return symbols_count + x + i * y;
+}
+
+td::uint32 FecHelper::get_left_fec_symbols_count() const {
+  auto fec_symbols_count = get_fec_symbols_count();
+  CHECK(fec_symbols_count > received_symbols_count);
+  return fec_symbols_count - received_symbols_count;
+}
+}  // namespace rldp2
+}  // namespace ton

rldp2/FecHelper.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "td/utils/int_types.h"
+
+namespace ton {
+namespace rldp2 {
+
+struct FecHelper {
+  td::uint32 symbols_count{0};
+  td::uint32 received_symbols_count{0};
+
+  td::uint32 get_fec_symbols_count() const;
+  td::uint32 get_left_fec_symbols_count() const;
+};
+
+}  // namespace rldp2
+}  // namespace ton

rldp2/InboundTransfer.cpp

@@ -0,0 +1,58 @@
+#include "InboundTransfer.h"
+
+#include "common/errorcode.h"
+
+namespace ton {
+namespace rldp2 {
+size_t InboundTransfer::total_size() const {
+  return data_.size();
+}
+
+std::map<td::uint32, InboundTransfer::Part> &InboundTransfer::parts() {
+  return parts_;
+}
+
+bool InboundTransfer::is_part_completed(td::uint32 part_i) {
+  return parts_.count(part_i) == 0 && part_i < next_part_;
+}
+
+td::Result<InboundTransfer::Part *> InboundTransfer::get_part(td::uint32 part_i, const ton::fec::FecType &fec_type) {
+  auto it = parts_.find(part_i);
+  if (it != parts_.end()) {
+    return &it->second;
+  }
+  //TODO: pass offset off and process even newer parts.
+  //LOG_CHECK(next_part_ >= part_i) << next_part_ << " >= " << part_i;
+  if (next_part_ == part_i && parts_.size() < 20) {
+    auto offset = offset_;
+    offset_ += fec_type.size();
+    if (offset_ > total_size()) {
+      return td::Status::Error(ErrorCode::protoviolation,
+                               PSTRING() << "too big part: offset=" << offset_ << " total_size=" << total_size()
+                                         << " total_size=" << fec_type.size() << " part=" << part_i);
+    }
+
+    auto decoder = fec_type.create_decoder().move_as_ok();
+    auto it = parts_.emplace(part_i, Part{std::move(decoder), RldpReceiver(RldpSender::Config()), offset});
+    next_part_++;
+    return &it.first->second;
+  }
+  return nullptr;
+}
+
+void InboundTransfer::finish_part(td::uint32 part_i, td::Slice data) {
+  auto it = parts_.find(part_i);
+  CHECK(it != parts_.end());
+  data_.as_slice().substr(it->second.offset).copy_from(data);
+  parts_.erase(it);
+}
+
+td::optional<td::Result<td::BufferSlice>> InboundTransfer::try_finish() {
+  if (parts_.empty() && offset_ == data_.size()) {
+    return std::move(data_);
+  }
+  return {};
+}
+
+}  // namespace rldp2
+}  // namespace ton

rldp2/InboundTransfer.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "td/utils/optional.h"
+
+#include "fec/fec.h"
+
+#include "RldpReceiver.h"
+
+#include <map>
+
+namespace ton {
+namespace rldp2 {
+struct InboundTransfer {
+  struct Part {
+    std::unique_ptr<td::fec::Decoder> decoder;
+    RldpReceiver receiver;
+    size_t offset;
+  };
+
+  explicit InboundTransfer(size_t total_size) : data_(total_size) {
+  }
+
+  size_t total_size() const;
+  std::map<td::uint32, Part> &parts();
+  bool is_part_completed(td::uint32 part_i);
+  td::Result<Part *> get_part(td::uint32 part_i, const ton::fec::FecType &fec_type);
+  void finish_part(td::uint32 part_i, td::Slice data);
+  td::optional<td::Result<td::BufferSlice>> try_finish();
+
+ private:
+  std::map<td::uint32, Part> parts_;
+  td::uint32 next_part_{0};
+  size_t offset_{0};
+  td::BufferSlice data_;
+};
+}  // namespace rldp2
+}  // namespace ton

rldp2/LossSender.cpp

@@ -0,0 +1,135 @@
+#include "LossSender.h"
+
+#include "td/utils/logging.h"
+
+#if TON_HAVE_GSL
+#include <gsl/gsl_cdf.h>
+#endif
+
+#include <cmath>
+
+namespace ton {
+namespace rldp2 {
+namespace {
+// works for 1e-x, where x in {1..10}
+double ndtri_fast(double p) {
+  if (p < 2e-10) {
+    return 6.361340902404;
+  }
+  if (p < 2e-9) {
+    return 5.997807015008;
+  }
+  if (p < 2e-8) {
+    return 5.612001244175;
+  }
+  if (p < 2e-7) {
+    return 5.199337582193;
+  }
+  if (p < 2e-6) {
+    return 4.753424308823;
+  }
+  if (p < 2e-5) {
+    return 4.264890793923;
+  }
+  if (p < 2e-4) {
+    return 3.719016485456;
+  }
+  if (p < 2e-3) {
+    return 3.090232306168;
+  }
+  if (p < 2e-2) {
+    return 2.326347874041;
+  }
+  return 1.281551565545;
+}
+}  // namespace
+
+LossSender::LossSender(double loss, double p) : loss_(loss), p_(p) {
+  v_.resize(2);
+  v_[0] = 1;
+  res_.push_back(0);
+  S_ = ndtri_fast(p_);
+  sigma_ = p * (1 - p);
+  //LOG(ERROR) << S_ << " " << ndtri(1 - p_);
+  //CHECK(fabs(S_ - ndtri(1 - p_)) < 1e-6);
+}
+
+int LossSender::send_n(int n) {
+  if (n < 50) {
+    return send_n_exact(n);
+  }
+  return send_n_approx_nbd(n);
+}
+
+int LossSender::send_n_exact(int n) {
+  while ((int)res_.size() <= n) {
+    step();
+  }
+  return res_[n];
+}
+
+int LossSender::send_n_approx_norm(int n) {
+  double a = (1 - loss_) * (1 - loss_);
+  double b = loss_ * (loss_ - 1) * (2 * n + S_ * S_);
+  double c = loss_ * loss_ * n * n + S_ * S_ * n * loss_ * (loss_ - 1);
+  double x = ((-b + sqrt(b * b - 4 * a * c)) / (2 * a));
+  return (int)(x + n + 1);
+}
+
+int LossSender::send_n_approx_nbd(int n) {
+#if TON_HAVE_GSL
+  auto mean = n * loss_ / (1 - loss_);
+  auto var = sqrt(mean / (1 - loss_));
+  auto min_k = static_cast<int>(mean + var);
+  auto max_k = min_k + static_cast<int>(var + 1) * 15;
+  while (min_k + 1 < max_k) {
+    int k = (min_k + max_k) / 2;
+    if (gsl_cdf_negative_binomial_P(k, 1 - loss_, n) > 1 - p_) {
+      max_k = k;
+    } else {
+      min_k = k;
+    }
+  }
+  return max_k + n;
+#endif
+  return send_n_approx_norm(n);
+}
+
+int LossSender::send_n_approx_pd(int n) {
+#if TON_HAVE_GSL
+  for (int k = 0;; k++) {
+    if (gsl_cdf_poisson_P(k, (n + k) * loss_) > 1 - p_) {
+      return k + n;
+    }
+  }
+#endif
+  return send_n_approx_norm(n);
+}
+bool LossSender::has_good_approx() {
+#if TON_HAVE_GSL
+  return true;
+#else
+  return false;
+#endif
+}
+
+void LossSender::step() {
+  n_++;
+  v_.push_back(0);
+  v_[n_] = v_[n_ - 1];
+  for (int j = n_; j >= 0; j--) {
+    v_[j + 1] += v_[j] * loss_;
+    v_[j] *= (1 - loss_);
+  }
+
+  while (res_i_ < n_ && v_[res_i_] < 1 - p_) {
+    res_i_++;
+  }
+  auto left_ = n_ - res_i_;
+  if ((int)res_.size() == left_) {
+    res_.push_back(n_);
+  }
+}
+
+}  // namespace rldp2
+}  // namespace ton

rldp2/LossSender.h

@@ -0,0 +1,31 @@
+#pragma once
+#include <vector>
+
+namespace ton {
+namespace rldp2 {
+struct LossSender {
+  LossSender(double loss, double p);
+  int send_n(int n);
+
+  int send_n_exact(int n);
+  int send_n_approx_norm(int n);
+  int send_n_approx_nbd(int n);
+  int send_n_approx_pd(int n);
+
+  bool has_good_approx();
+
+ private:
+  double loss_;
+  double p_;
+  double S_;
+  double sigma_;
+  int n_{0};
+  std::vector<double> v_;
+  std::vector<int> res_;
+  int res_i_{0};
+
+  void step();
+};
+
+}  // namespace rldp2
+}  // namespace ton

rldp2/LossStats.cpp

@@ -0,0 +1,23 @@
+#include "LossStats.h"
+#include "td/utils/misc.h"
+
+#include <cmath>
+
+namespace ton {
+namespace rldp2 {
+void LossStats::on_update(td::uint32 ack, td::uint32 lost) {
+  ack_ += ack;
+  lost_ += lost;
+
+  if (ack_ + lost_ > 1000) {
+    auto new_loss = td::clamp((double)lost_ / (ack_ + lost_), 0.001, 0.2);
+    if (fabs(new_loss - loss) > 5e-3) {
+      prob = LossSender(new_loss, 1e-9);
+    }
+    loss = new_loss;
+    ack_ = 0;
+    lost_ = 0;
+  }
+}
+}  // namespace rldp2
+}  // namespace ton

rldp2/LossStats.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "LossSender.h"
+
+#include "td/utils/int_types.h"
+
+namespace ton {
+namespace rldp2 {
+struct LossStats {
+  void on_update(td::uint32 ack, td::uint32 lost);
+  double loss = 0.1;
+  LossSender prob{0.1, 1e-9};
+
+ private:
+  td::uint32 ack_{0};
+  td::uint32 lost_{0};
+};
+}  // namespace rldp2
+}  // namespace ton

rldp2/OutboundTransfer.cpp

@@ -0,0 +1,40 @@
+#include "OutboundTransfer.h"
+
+namespace ton {
+namespace rldp2 {
+size_t OutboundTransfer::total_size() const {
+  return data_.size();
+}
+std::map<td::uint32, OutboundTransfer::Part> &OutboundTransfer::parts(const RldpSender::Config &config) {
+  while (parts_.size() < 20) {
+    auto offset = next_part_ * part_size();
+    if (offset >= data_.size()) {
+      break;
+    }
+    td::BufferSlice D = data_.from_slice(data_.as_slice().substr(offset).truncate(part_size()));
+    ton::fec::FecType fec_type = td::fec::RaptorQEncoder::Parameters{D.size(), symbol_size(), 0};
+    auto encoder = fec_type.create_encoder(std::move(D)).move_as_ok();
+    auto symbols_count = fec_type.symbols_count();
+    parts_.emplace(next_part_, Part{std::move(encoder), RldpSender(config, symbols_count), std::move(fec_type)});
+    next_part_++;
+  }
+  return parts_;
+}
+
+void OutboundTransfer::drop_part(td::uint32 part_i) {
+  parts_.erase(part_i);
+}
+
+OutboundTransfer::Part *OutboundTransfer::get_part(td::uint32 part_i) {
+  auto it = parts_.find(part_i);
+  if (it == parts_.end()) {
+    return nullptr;
+  }
+  return &it->second;
+}
+
+bool OutboundTransfer::is_done() const {
+  return next_part_ * part_size() >= data_.size() && parts_.empty();
+}
+}  // namespace rldp2
+}  // namespace ton

rldp2/OutboundTransfer.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "RldpSender.h"
+#include "fec/fec.h"
+
+#include <map>
+
+namespace ton {
+namespace rldp2 {
+struct OutboundTransfer {
+ public:
+  struct Part {
+    std::unique_ptr<td::fec::Encoder> encoder;
+    RldpSender sender;
+    ton::fec::FecType fec_type;
+  };
+
+  OutboundTransfer(td::BufferSlice data) : data_(std::move(data)) {
+  }
+
+  size_t total_size() const;
+  std::map<td::uint32, Part> &parts(const RldpSender::Config &config);
+  void drop_part(td::uint32 part_i);
+  Part *get_part(td::uint32 part_i);
+  bool is_done() const;
+
+ private:
+  td::BufferSlice data_;
+  std::map<td::uint32, Part> parts_;
+  td::uint32 next_part_{0};
+
+  static size_t part_size() {
+    return 2000000;
+  }
+  static size_t symbol_size() {
+    return 768;
+  }
+};
+}  // namespace rldp2
+}  // namespace ton

rldp2/Pacer.cpp

@@ -0,0 +1,45 @@
+#include "Pacer.h"
+namespace ton {
+namespace rldp2 {
+Pacer::Pacer(Options options)
+    : speed_(options.initial_speed)
+    , capacity_(options.initial_capacity)
+    , max_capacity_(options.max_capacity)
+    , time_granularity_(options.time_granularity) {
+}
+
+td::Timestamp Pacer::wakeup_at() const {
+  return wakeup_at_;
+}
+
+void Pacer::set_speed(double speed) {
+  if (speed < 1) {
+    speed = 1;
+  }
+  speed_ = speed;
+}
+
+td::optional<td::Timestamp> Pacer::send(double size, td::Timestamp now) {
+  update_capacity(now);
+
+  if (size < capacity_) {
+    capacity_ -= size;
+    return {};
+  }
+
+  size -= capacity_;
+  capacity_ = 0;
+  wakeup_at_ = td::Timestamp::in(size / speed_, now);
+  capacity_at_ = wakeup_at_;
+  return wakeup_at_;
+}
+
+void Pacer::update_capacity(td::Timestamp now) {
+  if (capacity_at_ && capacity_at_.is_in_past(now)) {
+    capacity_ += (now.at() - capacity_at_.at()) * speed_;
+    capacity_ = td::min(capacity_, td::max(max_capacity_, speed_ * time_granularity_));
+  }
+  capacity_at_ = now;
+}
+}  // namespace rldp2
+}  // namespace ton

rldp2/Pacer.h

@@ -0,0 +1,39 @@
+#pragma once
+#include "td/utils/optional.h"
+#include "td/utils/Time.h"
+
+namespace ton {
+namespace rldp2 {
+// NB: Should be careful with max_capacity < time_granularity * speed
+// We may send packet of any size.
+// After that we will be put to sleep till wakeup_at().
+// When we are awake we send packet of any size again.
+// Logic is - we don't have to wait to send a packet - it is poinless.
+// But we have to wait for some time after packet is sent
+class Pacer {
+ public:
+  struct Options {
+    Options() {
+    }
+    double initial_capacity{20};
+    double initial_speed{10};
+    double max_capacity{40};
+    double time_granularity{0.001};
+  };
+  Pacer(Options options = {});
+  td::Timestamp wakeup_at() const;
+  void set_speed(double speed);
+  td::optional<td::Timestamp> send(double size, td::Timestamp now);
+
+ private:
+  double speed_;
+  double capacity_;
+  double max_capacity_;
+  double time_granularity_;
+  td::Timestamp capacity_at_;
+  td::Timestamp wakeup_at_;
+
+  void update_capacity(td::Timestamp now);
+};
+}  // namespace rldp2
+}  // namespace ton

rldp2/RldpConnection.cpp

@@ -0,0 +1,369 @@
+#include "RldpConnection.h"
+
+#include "td/utils/overloaded.h"
+#include "td/utils/Random.h"
+#include "td/utils/tl_helpers.h"
+
+#include "tl-utils/tl-utils.hpp"
+#include "auto/tl/ton_api.h"
+#include "auto/tl/ton_api.hpp"
+
+#include "common/errorcode.h"
+
+#include "td/actor//actor.h"
+
+namespace ton {
+namespace rldp2 {
+void RldpConnection::add_limit(td::Timestamp timeout, Limit limit) {
+  CHECK(timeout);
+  auto p = limits_set_.insert(limit);
+  LOG_CHECK(p.second) << limit.transfer_id.to_hex();
+  limits_heap_.insert(timeout.at(), const_cast<Limit *>(&*p.first));
+}
+
+td::Timestamp RldpConnection::next_limit_expires_at() {
+  if (limits_heap_.empty()) {
+    return td::Timestamp::never();
+  }
+  return td::Timestamp::at(limits_heap_.top_key());
+}
+
+void RldpConnection::drop_limits(TransferId id) {
+  Limit limit;
+  limit.transfer_id = id;
+  auto it = limits_set_.find(limit);
+  if (it == limits_set_.end()) {
+    return;
+  }
+  limits_heap_.erase(const_cast<td::HeapNode *>(static_cast<const td::HeapNode *>(&*it)));
+  limits_set_.erase(it);
+}
+
+void RldpConnection::on_inbound_completed(TransferId transfer_id, td::Timestamp now) {
+  inbound_transfers_.erase(transfer_id);
+  completed_set_.insert(transfer_id);
+  completed_queue_.push(CompletedId{transfer_id, now.in(20)});
+  while (completed_queue_.size() > 128 && completed_queue_.front().timeout.is_in_past(now)) {
+    completed_set_.erase(completed_queue_.pop().transfer_id);
+  }
+}
+
+td::Timestamp RldpConnection::loop_limits(td::Timestamp now) {
+  while (!limits_heap_.empty() && td::Timestamp::at(limits_heap_.top_key()).is_in_past(now)) {
+    auto *limit = static_cast<Limit *>(limits_heap_.pop());
+    auto error = td::Status::Error(ErrorCode::timeout, "timeout");
+    if (limit->is_inbound) {
+      on_inbound_completed(limit->transfer_id, now);
+      to_receive_.emplace_back(limit->transfer_id, std::move(error));
+    } else {
+      auto it = outbound_transfers_.find(limit->transfer_id);
+      if (it != outbound_transfers_.end()) {
+        for (auto &part : it->second.parts(RldpSender::Config{})) {
+          in_flight_count_ -= part.second.sender.get_inflight_symbols_count();
+        }
+        outbound_transfers_.erase(it);
+        to_on_sent_.emplace_back(limit->transfer_id, std::move(error));
+      } else {
+        LOG(ERROR) << "Timeout on unknown transfer " << limit->transfer_id.to_hex();
+      }
+    }
+    limits_set_.erase(*limit);
+  }
+
+  return next_limit_expires_at();
+}
+
+void RldpConnection::set_receive_limits(TransferId transfer_id, td::Timestamp timeout, td::uint64 max_size) {
+  CHECK(timeout);
+  Limit limit;
+  limit.transfer_id = transfer_id;
+  limit.max_size = max_size;
+  limit.is_inbound = true;
+  add_limit(timeout, limit);
+}
+
+RldpConnection::RldpConnection() {
+  bdw_stats_.on_update(td::Timestamp::now(), 0);
+
+  rtt_stats_.windowed_min_rtt = 0.5;
+  bdw_stats_.windowed_max_bdw = 10;
+}
+
+void RldpConnection::send(TransferId transfer_id, td::BufferSlice data, td::Timestamp timeout) {
+  if (transfer_id.is_zero()) {
+    td::Random::secure_bytes(transfer_id.as_slice());
+  } else {
+    if (outbound_transfers_.find(transfer_id) != outbound_transfers_.end()) {
+      LOG(WARNING) << "Skip resend of " << transfer_id.to_hex();
+      return;
+    }
+  }
+
+  if (timeout) {
+    Limit limit;
+    limit.transfer_id = transfer_id;
+    limit.max_size = 0;
+    limit.is_inbound = false;
+    add_limit(timeout, limit);
+  }
+  outbound_transfers_.emplace(transfer_id, OutboundTransfer{std::move(data)});
+}
+
+void RldpConnection::receive_raw(td::BufferSlice packet) {
+  auto F = ton::fetch_tl_object<ton::ton_api::rldp2_MessagePart>(std::move(packet), true);
+  if (F.is_error()) {
+    return;
+  }
+  downcast_call(*F.move_as_ok(), [&](auto &obj) { this->receive_raw_obj(obj); });
+}
+
+void RldpConnection::loop_bbr(td::Timestamp now) {
+  bbr_.step(rtt_stats_, bdw_stats_, in_flight_count_, td::Timestamp::now());
+  //LOG(ERROR) << td::format::as_time(rtt_stats_.windowed_min_rtt) << " "
+  //<< td::format::as_size((td::int64)bdw_stats_.windowed_max_bdw * 768) << " " << rtt_stats_.rtt_round;
+  double speed = bbr_.get_rate();
+  td::uint32 congestion_window = bbr_.get_window_size();
+
+  static td::Timestamp next;
+  //FIXME: remove this UNSAFE debug output
+  if (next.is_in_past(now)) {
+    next = td::Timestamp::in(1, now);
+    if (td::actor::core::ActorExecuteContext::get()->actor().get_actor_info_ptr()->get_name() == "Alice") {
+      LOG(ERROR) << "speed=" << td::format::as_size((td::int64)speed * 768) << " "
+                 << "cgw=" << td::format::as_size((td::int64)congestion_window * 768) << " "
+                 << "loss=" << loss_stats_.loss * 100 << "%";
+    }
+  }
+
+  pacer_.set_speed(speed);
+  congestion_window_ = congestion_window;
+}
+
+td::Timestamp RldpConnection::run(ConnectionCallback &callback) {
+  auto now = td::Timestamp::now();
+  loop_bbr(now);
+
+  td::Timestamp alarm_timestamp;
+  td::VectorQueue<std::pair<const TransferId, OutboundTransfer> *> queue;
+  for (auto &outbound : outbound_transfers_) {
+    queue.push(&outbound);
+  }
+  while (!queue.empty()) {
+    auto outbound = queue.pop();
+    auto o_timeout = step(outbound->first, outbound->second, now);
+    if (o_timeout) {
+      alarm_timestamp.relax(o_timeout.unwrap());
+    } else {
+      queue.push(outbound);
+    }
+  }
+
+  if (in_flight_count_ > congestion_window_) {
+    bdw_stats_.on_pause(now);
+  }
+
+  for (auto &inbound : inbound_transfers_) {
+    alarm_timestamp.relax(run(inbound.first, inbound.second));
+  }
+
+  alarm_timestamp.relax(loop_limits(td::Timestamp::now()));
+
+  for (auto &data : to_receive_) {
+    callback.receive(data.first, std::move(data.second));
+  }
+  for (auto &raw : to_send_raw_) {
+    callback.send_raw(std::move(raw));
+  }
+  to_send_raw_.clear();
+  to_receive_.clear();
+  for (auto &res : to_on_sent_) {
+    callback.on_sent(res.first, std::move(res.second));
+  }
+  to_on_sent_.clear();
+
+  return alarm_timestamp;
+}
+
+td::Timestamp RldpConnection::run(const TransferId &transfer_id, InboundTransfer &inbound) {
+  td::Timestamp wakeup_at;
+  bool has_actions = true;
+  while (has_actions) {
+    has_actions = false;
+    for (auto &it : inbound.parts()) {
+      auto &inbound = it.second;
+      inbound.receiver.next_action(td::Timestamp::now())
+          .visit(td::overloaded([&](const RldpReceiver::ActionWait &wait) { wakeup_at.relax(wait.wait_till); },
+                                [&](const RldpReceiver::ActionSendAck &send) {
+                                  send_packet(ton::create_serialize_tl_object<ton::ton_api::rldp2_confirm>(
+                                      transfer_id, it.first, send.ack.max_seqno, send.ack.received_mask,
+                                      send.ack.received_count));
+                                  inbound.receiver.on_ack_sent(td::Timestamp::now());
+                                  has_actions = true;
+                                }));
+    }
+  }
+  return wakeup_at;
+}
+
+td::optional<td::Timestamp> RldpConnection::step(const TransferId &transfer_id, OutboundTransfer &outbound,
+                                                 td::Timestamp now) {
+  bool only_probe = in_flight_count_ > congestion_window_;
+
+  td::Timestamp wakeup_at;
+  if (!pacer_.wakeup_at().is_in_past(now)) {
+    wakeup_at = pacer_.wakeup_at();
+    only_probe = true;
+  }
+
+  for (auto &it : outbound.parts(RldpSender::Config{})) {
+    auto &part = it.second;
+
+    Guard guard(in_flight_count_, part.sender);
+    auto action = part.sender.next_action(now, only_probe);
+
+    bool was_send = false;
+    action.visit(td::overloaded(
+        [&](const RldpSender::ActionSend &send) {
+          auto seqno = send.seqno - 1;
+          if (part.encoder->get_info().ready_symbol_count <= seqno) {
+            part.encoder->prepare_more_symbols();
+          }
+          auto symbol = part.encoder->gen_symbol(seqno).data;
+          send_packet(ton::create_serialize_tl_object<ton::ton_api::rldp2_messagePart>(
+              transfer_id, part.fec_type.tl(), it.first, outbound.total_size(), seqno, std::move(symbol)));
+          if (!send.is_probe) {
+            pacer_.send(1, now);
+          }
+          part.sender.on_send(send.seqno, now, send.is_probe, rtt_stats_, bdw_stats_);
+          if (send.is_probe) {
+            //LOG(ERROR) << "PROBE " << it.first << " " << send.seqno;
+          }
+          //LOG(ERROR) << "SEND";
+          was_send = true;
+        },
+        [&](const RldpSender::ActionWait &wait) {
+          //LOG(ERROR) << "WAIT";
+          wakeup_at.relax(wait.wait_till);
+        }));
+    if (was_send) {
+      return {};
+    }
+  }
+
+  return wakeup_at;
+}
+
+void RldpConnection::receive_raw_obj(ton::ton_api::rldp2_messagePart &part) {
+  if (completed_set_.count(part.transfer_id_) > 0) {
+    send_packet(ton::create_serialize_tl_object<ton::ton_api::rldp2_complete>(part.transfer_id_, part.part_));
+    return;
+  }
+
+  auto r_total_size = td::narrow_cast_safe<td::size_t>(part.total_size_);
+  if (r_total_size.is_error()) {
+    return;
+  }
+  auto r_fec_type = ton::fec::FecType::create(std::move(part.fec_type_));
+  if (r_fec_type.is_error()) {
+    return;
+  }
+
+  auto total_size = r_total_size.move_as_ok();
+
+  auto transfer_id = part.transfer_id_;
+
+  // check total_size limits
+  td::uint64 max_size = default_mtu();
+  Limit key;
+  key.transfer_id = transfer_id;
+  auto limit_it = limits_set_.find(key);
+  bool has_limit = limit_it != limits_set_.end();
+  if (has_limit && limit_it->max_size != 0) {
+    max_size = limit_it->max_size;
+  }
+  if (total_size > max_size) {
+    LOG(INFO) << "Drop too big rldp query " << part.total_size_ << " > " << max_size;
+    return;
+  }
+
+  auto it = inbound_transfers_.find(transfer_id);
+  if (it == inbound_transfers_.end()) {
+    if (!has_limit) {
+      // set timeout even for small inbound queries
+      // TODO: other party stil may ddos us with small transfers
+      set_receive_limits(transfer_id, td::Timestamp::in(10), max_size);
+    }
+    it = inbound_transfers_.emplace(transfer_id, InboundTransfer{total_size}).first;
+  }
+
+  auto &inbound = it->second;
+  auto o_res = [&]() -> td::optional<td::Result<td::BufferSlice>> {
+    TRY_RESULT(in_part, inbound.get_part(part.part_, r_fec_type.move_as_ok()));
+    if (!in_part) {
+      if (inbound.is_part_completed(part.part_)) {
+        send_packet(ton::create_serialize_tl_object<ton::ton_api::rldp2_complete>(transfer_id, part.part_));
+      }
+      return {};
+    }
+    if (in_part->receiver.on_received(part.seqno_, td::Timestamp::now())) {
+      TRY_STATUS_PREFIX(in_part->decoder->add_symbol({static_cast<td::uint32>(part.seqno_), std::move(part.data_)}),
+                        td::Status::Error(ErrorCode::protoviolation, "invalid symbol"));
+      if (in_part->decoder->may_try_decode()) {
+        auto r_data = in_part->decoder->try_decode(false);
+        if (r_data.is_ok()) {
+          inbound.finish_part(part.part_, r_data.move_as_ok().data);
+        }
+      }
+    }
+    return inbound.try_finish();
+  }();
+
+  if (o_res) {
+    drop_limits(transfer_id);
+    on_inbound_completed(transfer_id, td::Timestamp::now());
+    to_receive_.emplace_back(transfer_id, o_res.unwrap());
+  }
+}
+
+void RldpConnection::receive_raw_obj(ton::ton_api::rldp2_complete &complete) {
+  auto transfer_id = complete.transfer_id_;
+  auto it = outbound_transfers_.find(transfer_id);
+  if (it == outbound_transfers_.end()) {
+    return;
+  }
+
+  auto *part = it->second.get_part(complete.part_);
+  if (part) {
+    in_flight_count_ -= part->sender.get_inflight_symbols_count();
+    it->second.drop_part(complete.part_);
+  }
+
+  if (it->second.is_done()) {
+    drop_limits(it->first);
+    to_on_sent_.emplace_back(it->first, td::Unit());
+    outbound_transfers_.erase(it);
+  }
+}
+
+void RldpConnection::receive_raw_obj(ton::ton_api::rldp2_confirm &confirm) {
+  auto transfer_id = confirm.transfer_id_;
+  auto it = outbound_transfers_.find(transfer_id);
+  if (it == outbound_transfers_.end()) {
+    return;
+  }
+  auto *part = it->second.get_part(confirm.part_);
+  if (!part) {
+    return;
+  }
+  Guard guard(in_flight_count_, part->sender);
+  Ack ack;
+  ack.max_seqno = confirm.max_seqno_;
+  ack.received_count = confirm.received_count_;
+  ack.received_mask = confirm.received_mask_;
+  auto update = part->sender.on_ack(ack, 0, td::Timestamp::now(), rtt_stats_, bdw_stats_, loss_stats_);
+  // update.new_received event
+  // update.o_loss_at event
+}
+
+}  // namespace rldp2
+}  // namespace ton

rldp2/RldpConnection.h

@@ -0,0 +1,119 @@
+#pragma once
+
+#include "Bbr.h"
+#include "InboundTransfer.h"
+#include "LossStats.h"
+#include "OutboundTransfer.h"
+#include "Pacer.h"
+#include "RttStats.h"
+
+#include "common/bitstring.h"
+
+#include "td/utils/buffer.h"
+#include "td/utils/Heap.h"
+#include "td/utils/VectorQueue.h"
+
+#include <set>
+
+namespace ton {
+namespace rldp2 {
+using TransferId = td::Bits256;
+class ConnectionCallback {
+ public:
+  virtual ~ConnectionCallback() {
+  }
+  virtual void send_raw(td::BufferSlice small_datagram) = 0;
+  virtual void receive(TransferId transfer_id, td::Result<td::BufferSlice> r_data) = 0;
+  virtual void on_sent(TransferId transfer_id, td::Result<td::Unit> state) = 0;
+};
+
+class RldpConnection {
+ public:
+  RldpConnection();
+  RldpConnection(RldpConnection &&other) = delete;
+  RldpConnection &operator=(RldpConnection &&other) = delete;
+  void send(TransferId tranfer_id, td::BufferSlice data, td::Timestamp timeout = td::Timestamp::never());
+  void set_receive_limits(TransferId transfer_id, td::Timestamp timeout, td::uint64 max_size);
+
+  void receive_raw(td::BufferSlice packet);
+
+  td::Timestamp run(ConnectionCallback &callback);
+
+  void set_default_mtu(td::uint64 mtu) {
+    default_mtu_ = mtu;
+  }
+  td::uint64 default_mtu() const {
+    return default_mtu_;
+  }
+
+ private:
+  td::uint64 default_mtu_ = 7680;
+
+  std::map<TransferId, OutboundTransfer> outbound_transfers_;
+  td::uint32 in_flight_count_{0};
+  std::map<TransferId, InboundTransfer> inbound_transfers_;
+
+  struct Limit : public td::HeapNode {
+    TransferId transfer_id;
+    td::uint64 max_size;
+    bool is_inbound;
+    bool operator<(const Limit &other) const {
+      return transfer_id < other.transfer_id;
+    }
+  };
+  td::KHeap<double> limits_heap_;
+  std::set<Limit> limits_set_;
+
+  struct CompletedId {
+    TransferId transfer_id;
+    td::Timestamp timeout;
+  };
+  td::VectorQueue<CompletedId> completed_queue_;
+  std::set<TransferId> completed_set_;
+
+  void add_limit(td::Timestamp timeout, Limit limit);
+  td::Timestamp next_limit_expires_at();
+  void drop_limits(TransferId id);
+  void on_inbound_completed(TransferId transfer_id, td::Timestamp now);
+  td::Timestamp loop_limits(td::Timestamp now);
+
+  void loop_bbr(td::Timestamp now);
+
+  RttStats rtt_stats_;
+  BdwStats bdw_stats_;
+  LossStats loss_stats_;
+  Bbr bbr_;
+  Pacer pacer_;
+  td::uint32 congestion_window_{0};
+
+  std::vector<td::BufferSlice> to_send_raw_;
+  std::vector<std::pair<TransferId, td::Result<td::BufferSlice>>> to_receive_;
+  std::vector<std::pair<TransferId, td::Result<td::Unit>>> to_on_sent_;
+
+  void send_packet(td::BufferSlice packet) {
+    to_send_raw_.push_back(std::move(packet));
+  };
+
+  td::Timestamp run(const TransferId &transfer_id, InboundTransfer &inbound);
+  struct Guard {
+    td::uint32 &in_flight_count;
+    const RldpSender &sender;
+    td::uint32 before_in_flight{sender.get_inflight_symbols_count()};
+
+    Guard(td::uint32 &in_flight_count, const RldpSender &sender) : in_flight_count(in_flight_count), sender(sender){};
+    ~Guard() {
+      in_flight_count -= before_in_flight;
+      in_flight_count += sender.get_inflight_symbols_count();
+    }
+  };
+
+  td::optional<td::Timestamp> step(const TransferId &transfer_id, OutboundTransfer &outbound, td::Timestamp now);
+
+  void receive_raw_obj(ton::ton_api::rldp2_messagePart &part);
+
+  void receive_raw_obj(ton::ton_api::rldp2_complete &part);
+
+  void receive_raw_obj(ton::ton_api::rldp2_confirm &part);
+};
+}  // namespace rldp2
+}  // namespace ton

rldp2/RldpReceiver.cpp

@@ -0,0 +1,34 @@
+#include "RldpReceiver.h"
+
+namespace ton {
+namespace rldp2 {
+td::Variant<RldpReceiver::ActionSendAck, RldpReceiver::ActionWait> RldpReceiver::next_action(td::Timestamp now) {
+  if (send_ack_at_ && (send_ack_at_.is_in_past(now))) {
+    return ActionSendAck{ack};
+  }
+  return ActionWait{send_ack_at_};
+}
+
+void RldpReceiver::on_ack_sent(td::Timestamp now) {
+  if (cnt_ != 0) {
+    //LOG(ERROR) << "RESEND ACK " << cnt_;
+  }
+  cnt_++;
+  if (cnt_ > 7) {
+    send_ack_at_ = {};
+  } else {
+    send_ack_at_.relax(td::Timestamp::at(now.at() + config_.ack_delay * (1 << cnt_)));
+  }
+}
+
+bool RldpReceiver::on_received(td::uint32 seqno, td::Timestamp now) {
+  if (!ack.on_got_packet(seqno)) {
+    return false;
+  }
+  cnt_ = 0;
+  send_ack_at_.relax(td::Timestamp::at(now.at() + config_.ack_delay));
+  return true;
+}
+
+}  // namespace rldp2
+}  // namespace ton

rldp2/RldpReceiver.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include "Ack.h"
+#include "RldpSender.h"
+
+namespace ton {
+namespace rldp2 {
+class RldpReceiver {
+ public:
+  RldpReceiver() = default;
+  RldpReceiver(RldpSender::Config config) : config_(config) {
+  }
+
+  struct ActionSendAck {
+    Ack ack;
+  };
+
+  struct ActionWait {
+    td::Timestamp wait_till;
+  };
+
+  td::Variant<ActionSendAck, ActionWait> next_action(td::Timestamp now);
+
+  bool on_received(td::uint32 seqno, td::Timestamp now);
+
+  void on_ack_sent(td::Timestamp now);
+
+ private:
+  Ack ack;
+  td::Timestamp send_ack_at_;
+  td::uint32 cnt_{0};
+
+  RldpSender::Config config_;
+};
+}  // namespace rldp2
+}  // namespace ton

rldp2/RldpSender.cpp

@@ -0,0 +1,95 @@
+#include "RldpSender.h"
+
+#include "RttStats.h"
+#include "LossStats.h"
+#include "BdwStats.h"
+
+#include "td/utils/misc.h"
+
+namespace ton {
+namespace rldp2 {
+td::Variant<RldpSender::ActionWait, RldpSender::ActionSend> RldpSender::next_action(td::Timestamp now,
+                                                                                    bool only_probe) {
+  if (!only_probe && extra_symbols_ > get_inflight_symbols_count()) {
+    //LOG(ERROR) << fec_helper_.symbols_count << " " << fec_helper_.get_extra_symbols_count();
+    return ActionSend{packets_.next_seqno(), false};
+  }
+
+  return next_probe(now);
+}
+
+td::Variant<RldpSender::ActionWait, RldpSender::ActionSend> RldpSender::next_probe(td::Timestamp now) {
+  if (probe_timeout_.is_in_past(now)) {
+    return ActionSend{packets_.next_seqno(), true};
+  }
+  return ActionWait{probe_timeout_};
+}
+
+SenderPackets::Update RldpSender::on_ack(const Ack &ack, double ack_delay, td::Timestamp now, RttStats &rtt_stats,
+                                         BdwStats &bdw_stats, LossStats &loss_stats) {
+  //LOG(ERROR) << "ON ACK " << ack.max_seqno << " " << ack.received_mask << " " << ack.received_count;
+  auto update = packets_.on_ack(ack);
+  if (!update.was_max_updated) {
+    CHECK(!update.new_received);
+    return update;
+  }
+
+  // update rtt
+  ack_delay = td::clamp(ack_delay, 0.0, config_.max_ack_delay);
+  auto rtt_sample = now.at() - packets_.max_packet().sent_at.at();
+  rtt_stats.on_rtt_sample(rtt_sample, ack_delay, now);
+
+  bdw_stats.on_update(now, update.new_received);
+  bdw_stats.on_packet_ack(packets_.max_packet().bdw_packet_info, packets_.max_packet().sent_at, now);
+
+  // drop ready packets
+  SenderPackets::Limits limits;
+  limits.sent_at = td::Timestamp::at(now.at() - get_loss_delay(rtt_stats));
+  limits.seqno = sub_or_zero(packets_.max_packet().seqno, get_loss_seqno_delay());
+  update.drop_update = packets_.drop_packets(limits);
+
+  loss_stats.on_update(update.drop_update.new_ack, update.drop_update.new_lost);
+
+  fec_helper_.received_symbols_count = packets_.received_count();
+  extra_symbols_ = loss_stats.prob.send_n(fec_helper_.get_left_fec_symbols_count());
+  return update;
+}
+
+void RldpSender::on_send(td::uint32 seqno, td::Timestamp now, bool is_probe, const RttStats &rtt_stats,
+                         const BdwStats &bdw_stats) {
+  SenderPackets::Packet packet;
+  packet.is_in_flight = true;
+  packet.sent_at = now;
+  packet.seqno = seqno;
+  packet.size = 0;
+  packet.bdw_packet_info = bdw_stats.on_packet_send(packets_.first_sent_at(now));
+  packets_.send(packet);
+
+  probe_timeout_ = td::Timestamp::at(now.at() + get_probe_delay(rtt_stats));
+
+  if (is_probe) {
+    //LOG(ERROR) << get_probe_delay(rtt_stats) << " " << rtt_stats.last_rtt << " " << packets_.in_flight_count() << " "
+    //<< packets_.received_count();
+    probe_k_ = std::min(probe_k_ * 2, 10u);
+  } else {
+    probe_k_ = 1;
+  }
+}
+
+double RldpSender::get_loss_delay(const RttStats &rtt_stats) {
+  auto rtt = std::max(rtt_stats.last_rtt, rtt_stats.smoothed_rtt);
+  if (rtt < 0) {
+    rtt = config_.initial_rtt;
+  }
+  return rtt * 8 / 7;
+}
+
+double RldpSender::get_probe_delay(const RttStats &rtt_stats) {
+  if (rtt_stats.last_rtt < 0) {
+    return config_.initial_rtt * 2;
+  } else {
+    return (rtt_stats.smoothed_rtt + rtt_stats.rtt_var * 4 + config_.max_ack_delay) * probe_k_;
+  }
+}
+}  // namespace rldp2
+}  // namespace ton

rldp2/RldpSender.h

@@ -0,0 +1,81 @@
+#pragma once
+
+#include "td/utils/Time.h"
+#include "td/utils/Variant.h"
+
+#include "FecHelper.h"
+#include "SenderPackets.h"
+
+namespace ton {
+namespace rldp2 {
+struct Ack;
+struct BdwStats;
+struct RttStats;
+struct LossStats;
+
+inline td::uint32 sub_or_zero(td::uint32 a, td::uint32 b) {
+  if (a < b) {
+    return 0;
+  }
+  return a - b;
+}
+
+class RldpSender {
+ public:
+  struct Config {
+    static constexpr double DEFAULT_MAX_ACK_DELAY = 0.01;
+    static constexpr td::uint32 DEFAULT_PACKET_TRESHOLD = 3;
+    static constexpr double DEFAULT_INITIAL_RTT = 0.5;
+
+    double max_ack_delay{DEFAULT_MAX_ACK_DELAY};
+    double ack_delay{DEFAULT_MAX_ACK_DELAY};
+    td::uint32 packet_treshold{DEFAULT_PACKET_TRESHOLD};
+    double initial_rtt{DEFAULT_INITIAL_RTT};
+  };
+
+  RldpSender() = default;
+  RldpSender(Config config, td::uint32 symbols_count) : config_(config) {
+    fec_helper_.symbols_count = symbols_count;
+    extra_symbols_ = fec_helper_.get_left_fec_symbols_count();
+  }
+
+  struct ActionWait {
+    td::Timestamp wait_till;
+  };
+
+  struct ActionSend {
+    td::uint32 seqno;
+    bool is_probe;
+  };
+
+  td::Variant<ActionWait, ActionSend> next_action(td::Timestamp now, bool only_probe = false);
+  td::Variant<ActionWait, ActionSend> next_probe(td::Timestamp now);
+
+  td::uint32 get_inflight_symbols_count() const {
+    return packets_.in_flight_count();
+  }
+
+  SenderPackets::Update on_ack(const Ack &ack, double ack_delay, td::Timestamp now, RttStats &rtt_stats,
+                               BdwStats &bdw_stats, LossStats &loss_stats);
+
+  void on_send(td::uint32 seqno, td::Timestamp now, bool is_probe, const RttStats &rtt_stats,
+               const BdwStats &bdw_state);
+
+ private:
+  Config config_;
+  SenderPackets packets_;
+  FecHelper fec_helper_;
+  td::Timestamp probe_timeout_;
+  td::uint32 probe_k_{1};
+  td::uint32 extra_symbols_{0};
+
+  double get_loss_delay(const RttStats &rtt_stats);
+
+  double get_probe_delay(const RttStats &rtt_stats);
+
+  td::uint32 get_loss_seqno_delay() {
+    return config_.packet_treshold;
+  }
+};
+}  // namespace rldp2
+}  // namespace ton

rldp2/RttStats.cpp

@@ -0,0 +1,51 @@
+#include "RttStats.h"
+#include <cmath>
+
+namespace ton {
+namespace rldp2 {
+void RttStats::on_rtt_sample(double rtt_sample, double ack_delay, td::Timestamp now) {
+  if (rtt_sample < 0.001 || rtt_sample > 10) {
+    LOG(WARNING) << "Suspicious rtt sample " << rtt_sample;
+    return;
+  }
+  if (ack_delay < -1e-9 || ack_delay > 10) {
+    LOG(WARNING) << "Suspicious ack_delay " << ack_delay;
+    return;
+  }
+  rtt_sample = td::max(0.01, rtt_sample);
+
+  last_rtt = rtt_sample;
+
+  windowed_min_rtt_stat.add_event(rtt_sample, now.at());
+  auto windowed_min_rtt_sample = windowed_min_rtt_stat.get_stat(now.at()).get_stat();
+  if (windowed_min_rtt_sample) {
+    windowed_min_rtt = windowed_min_rtt_sample.value();
+  }
+
+  if (smoothed_rtt < 0) {
+    // ignore ack_delay just because
+    min_rtt = last_rtt;
+    smoothed_rtt = last_rtt;
+    rtt_var = last_rtt / 2;
+  } else {
+    if (rtt_sample < min_rtt) {
+      min_rtt = rtt_sample;
+    }
+
+    double adjusted_rtt = rtt_sample;
+    if (adjusted_rtt - ack_delay > min_rtt) {
+      adjusted_rtt -= ack_delay;
+    }
+
+    smoothed_rtt += (adjusted_rtt - smoothed_rtt) / 8;
+    double var = fabs(smoothed_rtt - adjusted_rtt);
+    rtt_var += (var - rtt_var) / 4;
+  }
+
+  if (td::Timestamp::in(smoothed_rtt, rtt_round_at).is_in_past(now)) {
+    rtt_round_at = now;
+    rtt_round++;
+  }
+}
+}  // namespace rldp2
+}  // namespace ton

rldp2/RttStats.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "td/utils/Time.h"
+#include "td/utils/TimedStat.h"
+
+namespace ton {
+namespace rldp2 {
+struct RttStats {
+  void on_rtt_sample(double rtt_sample, double ack_delay, td::Timestamp now);
+
+  double min_rtt = -1;
+  double windowed_min_rtt = -1;
+  double last_rtt = -1;
+  double smoothed_rtt = -1;
+  double rtt_var = -1;
+  td::uint32 rtt_round{0};
+
+ private:
+  td::Timestamp rtt_round_at;
+  td::TimedStat<td::MinStat<double>> windowed_min_rtt_stat{5, 0};
+};
+}  // namespace rldp2
+}  // namespace ton

rldp2/SenderPackets.cpp

@@ -0,0 +1,125 @@
+#include "SenderPackets.h"
+
+#include "td/utils/bits.h"
+
+namespace ton {
+namespace rldp2 {
+td::uint32 SenderPackets::next_seqno() const {
+  return last_seqno_ + 1;
+}
+
+SenderPackets::DropUpdate SenderPackets::drop_packets(const Limits &limits) {
+  while (!packets.empty()) {
+    auto &packet = packets.front();
+    if (!limits.should_drop(packet)) {
+      break;
+    }
+    mark_ack_or_lost(packet);
+    packets.pop();
+  }
+  DropUpdate update;
+  update.new_ack = total_ack_ - last_total_ack_;
+  update.new_lost = total_lost_ - last_total_lost_;
+  last_total_ack_ = total_ack_;
+  last_total_lost_ = total_lost_;
+  update.o_loss_at = std::move(last_loss_);
+  return update;
+}
+
+SenderPackets::Update SenderPackets::on_ack(Ack ack) {
+  ack.max_seqno = td::min(ack.max_seqno, last_seqno_);
+  ack.received_count = td::min(ack.received_count, ack.max_seqno);
+
+  // TODO: seqno of rldp and seqno of a packet must be completly separate seqnos
+  Update update;
+  if (received_count_ < ack.received_count) {
+    update.new_received = ack.received_count - received_count_;
+    left_ack_ += update.new_received;
+    left_ack_ = td::min(left_ack_, in_flight_count_);
+    received_count_ = ack.received_count;
+  }
+
+  if (max_packet_.seqno > ack.max_seqno) {
+    return update;
+  }
+
+  auto packet = get_packet(ack.max_seqno);
+  if (!packet) {
+    return update;
+  }
+
+  if (max_packet_.seqno < ack.max_seqno) {
+    update.was_max_updated = true;
+    max_packet_ = *packet;
+  }
+
+  for (td::uint32 i : td::BitsRange(ack.received_mask)) {
+    if (ack.max_seqno < i) {
+      break;
+    }
+    auto seqno = ack.max_seqno - i;
+    auto packet = get_packet(seqno);
+    if (!packet) {
+      break;
+    }
+    mark_ack(*packet);
+  }
+
+  return update;
+}
+void SenderPackets::mark_ack_or_lost(Packet &packet) {
+  if (left_ack_) {
+    mark_ack(packet);
+  } else {
+    mark_lost(packet);
+  }
+}
+
+void SenderPackets::mark_lost(Packet &packet) {
+  if (!packet.is_in_flight) {
+    return;
+  }
+  total_lost_++;
+  in_flight_count_--;
+  packet.is_in_flight = false;
+  last_loss_ = packet.sent_at;
+}
+
+void SenderPackets::mark_ack(Packet &packet) {
+  if (!packet.is_in_flight) {
+    return;
+  }
+  if (left_ack_ > 0) {
+    left_ack_--;
+  }
+  total_ack_++;
+  in_flight_count_--;
+  packet.is_in_flight = false;
+}
+
+SenderPackets::Packet *SenderPackets::get_packet(td::uint32 seqno) {
+  if (packets.empty()) {
+    return nullptr;
+  }
+  auto front_seqno = packets.front().seqno;
+  if (front_seqno > seqno) {
+    return nullptr;
+  }
+  td::uint32 index = seqno - front_seqno;
+  if (index >= packets.size()) {
+    return nullptr;
+  }
+  auto packet = packets.data() + index;
+  CHECK(packet->seqno == seqno);
+  return packet;
+}
+
+void SenderPackets::send(Packet packet) {
+  CHECK(next_seqno() == packet.seqno);
+  packets.push(packet);
+  last_seqno_++;
+  in_flight_count_ += packet.is_in_flight;
+}
+
+}  // namespace rldp2
+}  // namespace ton

rldp2/SenderPackets.h

@@ -0,0 +1,89 @@
+#pragma once
+
+#include "td/utils/VectorQueue.h"
+#include "Ack.h"
+#include "BdwStats.h"
+
+namespace ton {
+namespace rldp2 {
+class SenderPackets {
+ public:
+  struct Packet {
+    bool is_in_flight{false};
+    td::Timestamp sent_at;
+    td::uint32 seqno{0};
+    td::uint32 size{0};
+
+    BdwStats::PacketInfo bdw_packet_info;
+  };
+
+  struct Limits {
+    td::Timestamp sent_at;
+    td::uint32 seqno{0};
+    bool should_drop(const Packet &packet) const {
+      return !packet.is_in_flight || packet.sent_at < sent_at || packet.seqno < seqno;
+    }
+  };
+
+  struct DropUpdate {
+    td::uint32 new_ack{0};  // ~= new_received
+    td::uint32 new_lost{0};
+    td::optional<td::Timestamp> o_loss_at;
+  };
+
+  struct Update {
+    bool was_max_updated{false};
+    td::uint32 new_received{0};
+
+    DropUpdate drop_update;
+  };
+
+  td::VectorQueue<Packet> packets;
+
+  void send(Packet packet);
+
+  td::uint32 next_seqno() const;
+  DropUpdate drop_packets(const Limits &limits);
+
+  Update on_ack(Ack ack);
+
+  td::uint32 in_flight_count() const {
+    return in_flight_count_;
+  }
+  td::uint32 received_count() const {
+    return received_count_;
+  }
+  const Packet &max_packet() const {
+    return max_packet_;
+  }
+  td::Timestamp first_sent_at(td::Timestamp now) const {
+    if (!packets.empty()) {
+      now.relax(packets.front().sent_at);
+    }
+    return now;
+  }
+
+ private:
+  td::uint32 in_flight_count_{0};  // sum(packet.is_in_flight for packet in packets)
+  td::uint32 received_count_{0};
+  td::uint32 last_seqno_{0};
+  Packet max_packet_;
+
+  td::uint32 total_ack_{0};
+  td::uint32 total_lost_{0};
+  td::uint32 last_total_ack_{0};
+  td::uint32 last_total_lost_{0};
+
+  td::optional<td::Timestamp> last_loss_;
+  td::uint32 left_ack_{0};
+
+  void mark_ack_or_lost(Packet &packet);
+
+  void mark_lost(Packet &packet);
+
+  void mark_ack(Packet &packet);
+
+  Packet *get_packet(td::uint32 seqno);
+};
+}  // namespace rldp2
+}  // namespace ton

rldp2/rldp-in.hpp

@@ -0,0 +1,113 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "rldp.hpp"
+
+#include "tl-utils/tl-utils.hpp"
+#include "adnl/adnl-query.h"
+#include "adnl/adnl-peer-table.h"
+
+#include "td/utils/List.h"
+
+#include <map>
+#include <set>
+
+namespace ton {
+
+namespace rldp2 {
+
+class RldpLru : public td::ListNode {
+ public:
+  TransferId transfer_id() {
+    return transfer_id_;
+  }
+
+  RldpLru(TransferId transfer_id) : transfer_id_(transfer_id) {
+  }
+  RldpLru() {
+  }
+
+  static RldpLru *from_list_node(td::ListNode *node) {
+    return static_cast<RldpLru *>(node);
+  }
+
+ private:
+  TransferId transfer_id_;
+};
+
+class RldpConnectionActor;
+class RldpIn : public RldpImpl {
+ public:
+  static constexpr td::uint64 mtu() {
+    return (1ull << 37);
+  }
+  static constexpr td::uint32 lru_size() {
+    return 128;
+  }
+  void on_sent(TransferId transfer_id, td::Result<td::Unit> state);
+
+  void send_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override;
+  void send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout,
+                       td::BufferSlice data) override;
+
+  void send_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name,
+                  td::Promise<td::BufferSlice> promise, td::Timestamp timeout, td::BufferSlice data) override {
+    send_query_ex(src, dst, name, std::move(promise), timeout, std::move(data), default_mtu());
+  }
+  void send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name,
+                     td::Promise<td::BufferSlice> promise, td::Timestamp timeout, td::BufferSlice data,
+                     td::uint64 max_answer_size) override;
+  void answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout,
+                    adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data);
+
+  void receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data);
+
+  void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id,
+                       ton_api::rldp_message &message);
+  void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id,
+                       ton_api::rldp_query &message);
+  void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id,
+                       ton_api::rldp_answer &message);
+  void receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id,
+                       td::Result<td::BufferSlice> data);
+
+  void add_id(adnl::AdnlNodeIdShort local_id) override;
+
+  RldpIn(td::actor::ActorId<adnl::AdnlPeerTable> adnl) : adnl_(adnl) {
+  }
+
+ private:
+  std::unique_ptr<adnl::Adnl::Callback> make_adnl_callback();
+
+  td::actor::ActorId<adnl::AdnlPeerTable> adnl_;
+
+  std::map<std::pair<adnl::AdnlNodeIdShort, adnl::AdnlNodeIdShort>, td::actor::ActorOwn<RldpConnectionActor>>
+      connections_;
+
+  std::map<TransferId, td::Promise<td::BufferSlice>> queries_;
+
+  std::set<adnl::AdnlNodeIdShort> local_ids_;
+
+  td::actor::ActorId<RldpConnectionActor> create_connection(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst);
+};
+
+}  // namespace rldp2
+
+}  // namespace ton

rldp2/rldp.cpp

@@ -0,0 +1,246 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "rldp-in.hpp"
+#include "auto/tl/ton_api.h"
+#include "auto/tl/ton_api.hpp"
+#include "td/utils/Random.h"
+#include "fec/fec.h"
+#include "RldpConnection.h"
+
+namespace ton {
+
+namespace rldp2 {
+
+class RldpConnectionActor : public td::actor::Actor, private ConnectionCallback {
+ public:
+  RldpConnectionActor(td::actor::ActorId<RldpIn> rldp, adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst,
+                      td::actor::ActorId<adnl::Adnl> adnl)
+      : rldp_(std::move(rldp)), src_(src), dst_(dst), adnl_(std::move(adnl)){};
+
+  void send(TransferId transfer_id, td::BufferSlice query, td::Timestamp timeout = td::Timestamp::never()) {
+    connection_.send(transfer_id, std::move(query), timeout);
+    yield();
+  }
+  void set_receive_limits(TransferId transfer_id, td::Timestamp timeout, td::uint64 max_size) {
+    connection_.set_receive_limits(transfer_id, timeout, max_size);
+  }
+  void receive_raw(td::BufferSlice data) {
+    connection_.receive_raw(std::move(data));
+    yield();
+  }
+
+ private:
+  td::actor::ActorId<RldpIn> rldp_;
+  adnl::AdnlNodeIdShort src_;
+  adnl::AdnlNodeIdShort dst_;
+  td::actor::ActorId<adnl::Adnl> adnl_;
+  RldpConnection connection_;
+
+  void loop() override {
+    alarm_timestamp() = connection_.run(*this);
+  }
+
+  void send_raw(td::BufferSlice data) override {
+    send_closure(adnl_, &adnl::Adnl::send_message, src_, dst_, std::move(data));
+  }
+  void receive(TransferId transfer_id, td::Result<td::BufferSlice> data) override {
+    send_closure(rldp_, &RldpIn::receive_message, dst_, src_, transfer_id, std::move(data));
+  }
+  void on_sent(TransferId transfer_id, td::Result<td::Unit> state) override {
+    send_closure(rldp_, &RldpIn::on_sent, transfer_id, std::move(state));
+  }
+};
+
+namespace {
+TransferId get_random_transfer_id() {
+  TransferId transfer_id;
+  td::Random::secure_bytes(transfer_id.as_slice());
+  return transfer_id;
+}
+TransferId get_responce_transfer_id(TransferId transfer_id) {
+  return transfer_id ^ TransferId::ones();
+}
+}  // namespace
+
+void RldpIn::send_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) {
+  return send_message_ex(src, dst, td::Timestamp::in(10.0), std::move(data));
+}
+
+void RldpIn::send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout,
+                             td::BufferSlice data) {
+  td::Bits256 id;
+  td::Random::secure_bytes(id.as_slice());
+
+  auto B = serialize_tl_object(create_tl_object<ton_api::rldp_message>(id, std::move(data)), true);
+
+  auto transfer_id = get_random_transfer_id();
+  send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout);
+}
+
+void RldpIn::send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name,
+                           td::Promise<td::BufferSlice> promise, td::Timestamp timeout, td::BufferSlice data,
+                           td::uint64 max_answer_size) {
+  auto query_id = adnl::AdnlQuery::random_query_id();
+
+  auto date = static_cast<td::uint32>(timeout.at_unix()) + 1;
+  auto B = serialize_tl_object(create_tl_object<ton_api::rldp_query>(query_id, max_answer_size, date, std::move(data)),
+                               true);
+
+  auto connection = create_connection(src, dst);
+  auto transfer_id = get_random_transfer_id();
+  auto response_transfer_id = get_responce_transfer_id(transfer_id);
+  send_closure(connection, &RldpConnectionActor::set_receive_limits, response_transfer_id, timeout, max_answer_size);
+  send_closure(connection, &RldpConnectionActor::send, transfer_id, std::move(B), timeout);
+
+  queries_.emplace(response_transfer_id, std::move(promise));
+}
+
+void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout,
+                          adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data) {
+  auto B = serialize_tl_object(create_tl_object<ton_api::rldp_answer>(query_id, std::move(data)), true);
+
+  send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout);
+}
+
+void RldpIn::receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data) {
+  send_closure(create_connection(local_id, source), &RldpConnectionActor::receive_raw, std::move(data));
+}
+
+td::actor::ActorId<RldpConnectionActor> RldpIn::create_connection(adnl::AdnlNodeIdShort src,
+                                                                  adnl::AdnlNodeIdShort dst) {
+  auto it = connections_.find(std::make_pair(src, dst));
+  if (it != connections_.end()) {
+    return it->second.get();
+  }
+  auto connection = td::actor::create_actor<RldpConnectionActor>("RldpConnection", actor_id(this), src, dst, adnl_);
+  auto res = connection.get();
+  connections_[std::make_pair(src, dst)] = std::move(connection);
+  return res;
+}
+
+void RldpIn::receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id,
+                             td::Result<td::BufferSlice> r_data) {
+  if (r_data.is_error()) {
+    auto it = queries_.find(transfer_id);
+    if (it != queries_.end()) {
+      it->second.set_error(r_data.move_as_error());
+      queries_.erase(it);
+    } else {
+      VLOG(RLDP_INFO) << "received error to unknown transfer_id " << transfer_id << " " << r_data.error();
+    }
+    return;
+  }
+
+  auto data = r_data.move_as_ok();
+  //LOG(ERROR) << "RECEIVE MESSAGE " << data.size();
+  auto F = fetch_tl_object<ton_api::rldp_Message>(std::move(data), true);
+  if (F.is_error()) {
+    VLOG(RLDP_INFO) << "failed to parse rldp packet [" << source << "->" << local_id << "]: " << F.move_as_error();
+    return;
+  }
+
+  ton_api::downcast_call(*F.move_as_ok().get(),
+                         [&](auto &obj) { this->process_message(source, local_id, transfer_id, obj); });
+}
+
+void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id,
+                             ton_api::rldp_message &message) {
+  td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::deliver, source, local_id, std::move(message.data_));
+}
+
+void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id,
+                             ton_api::rldp_query &message) {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), source, local_id,
+                                       timeout = td::Timestamp::at_unix(message.timeout_), query_id = message.query_id_,
+                                       max_answer_size = static_cast<td::uint64>(message.max_answer_size_),
+                                       transfer_id](td::Result<td::BufferSlice> R) {
+    if (R.is_ok()) {
+      auto data = R.move_as_ok();
+      if (data.size() > max_answer_size) {
+        VLOG(RLDP_NOTICE) << "rldp query failed: answer too big";
+      } else {
+        td::actor::send_closure(SelfId, &RldpIn::answer_query, local_id, source, timeout, query_id,
+                                transfer_id ^ TransferId::ones(), std::move(data));
+      }
+    } else {
+      VLOG(RLDP_NOTICE) << "rldp query failed: " << R.move_as_error();
+    }
+  });
+  VLOG(RLDP_DEBUG) << "delivering rldp query";
+  td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::deliver_query, source, local_id, std::move(message.data_),
+                          std::move(P));
+}
+
+void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id,
+                             ton_api::rldp_answer &message) {
+  auto it = queries_.find(transfer_id);
+  if (it != queries_.end()) {
+    it->second.set_value(std::move(message.data_));
+    queries_.erase(it);
+  } else {
+    VLOG(RLDP_INFO) << "received answer to unknown query " << message.query_id_;
+  }
+}
+
+void RldpIn::on_sent(TransferId transfer_id, td::Result<td::Unit> state) {
+  //TODO: completed transfer
+}
+
+void RldpIn::add_id(adnl::AdnlNodeIdShort local_id) {
+  if (local_ids_.count(local_id) == 1) {
+    return;
+  }
+
+  std::vector<std::string> X{adnl::Adnl::int_to_bytestring(ton_api::rldp2_messagePart::ID),
+                             adnl::Adnl::int_to_bytestring(ton_api::rldp2_confirm::ID),
+                             adnl::Adnl::int_to_bytestring(ton_api::rldp2_complete::ID)};
+  for (auto &x : X) {
+    td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, x, make_adnl_callback());
+  }
+
+  local_ids_.insert(local_id);
+}
+
+std::unique_ptr<adnl::Adnl::Callback> RldpIn::make_adnl_callback() {
+  class Callback : public adnl::Adnl::Callback {
+   private:
+    td::actor::ActorId<RldpIn> id_;
+
+   public:
+    Callback(td::actor::ActorId<RldpIn> id) : id_(id) {
+    }
+    void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override {
+      td::actor::send_closure(id_, &RldpIn::receive_message_part, src, dst, std::move(data));
+    }
+    void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data,
+                       td::Promise<td::BufferSlice> promise) override {
+      promise.set_error(td::Status::Error(ErrorCode::notready, "rldp does not support queries"));
+    }
+  };
+
+  return std::make_unique<Callback>(actor_id(this));
+}
+
+td::actor::ActorOwn<Rldp> Rldp::create(td::actor::ActorId<adnl::Adnl> adnl) {
+  return td::actor::create_actor<RldpIn>("rldp", td::actor::actor_dynamic_cast<adnl::AdnlPeerTable>(adnl));
+}
+
+}  // namespace rldp2
+
+}  // namespace ton

rldp2/rldp.h

@@ -0,0 +1,45 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2020 Telegram Systems LLP
+*/
+#pragma once
+
+#include "adnl/adnl.h"
+
+namespace ton {
+
+namespace rldp2 {
+
+class Rldp : public adnl::AdnlSenderInterface {
+ public:
+  virtual ~Rldp() = default;
+
+  static constexpr td::uint64 default_mtu() {
+    return adnl::Adnl::get_mtu();
+  }
+
+  virtual void add_id(adnl::AdnlNodeIdShort local_id) = 0;
+
+  virtual void send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout,
+                               td::BufferSlice data) = 0;
+
+  static td::actor::ActorOwn<Rldp> create(td::actor::ActorId<adnl::Adnl> adnl);
+};
+
+}  // namespace rldp2
+
+}  // namespace ton

rldp2/rldp.hpp

@@ -0,0 +1,48 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "rldp.h"
+
+#include "tl-utils/tl-utils.hpp"
+#include "adnl/adnl-query.h"
+
+#include <map>
+
+namespace ton {
+
+namespace rldp2 {
+
+constexpr int VERBOSITY_NAME(RLDP_WARNING) = verbosity_WARNING;
+constexpr int VERBOSITY_NAME(RLDP_NOTICE) = verbosity_INFO;
+constexpr int VERBOSITY_NAME(RLDP_INFO) = verbosity_DEBUG;
+constexpr int VERBOSITY_NAME(RLDP_DEBUG) = verbosity_DEBUG;
+constexpr int VERBOSITY_NAME(RLDP_EXTRA_DEBUG) = verbosity_DEBUG + 1;
+
+using TransferId = td::Bits256;
+
+class RldpImpl : public Rldp {
+ public:
+  //virtual void transfer_completed(TransferId transfer_id) = 0;
+  //virtual void in_transfer_completed(TransferId transfer_id) = 0;
+};
+
+}  // namespace rldp2
+
+}  // namespace ton

storage/Bitset.h

@@ -0,0 +1,69 @@
+#pragma once
+
+#include "td/utils/Slice.h"
+#include "td/utils/logging.h"
+
+namespace td {
+struct Bitset {
+ public:
+  td::Slice as_slice() const {
+    return td::Slice(bits_).substr(0, (bits_size_ + 7) / 8);
+  }
+  bool get(size_t offset) const {
+    auto i = offset / 8;
+    if (i >= bits_.size()) {
+      return false;
+    }
+    auto bit_i = offset % 8;
+    return (bits_[i] & (1 << bit_i)) != 0;
+  }
+
+  void reserve(size_t offset) {
+    auto i = offset / 8;
+    if (i >= bits_.size()) {
+      bits_.resize(i + 1);
+    }
+  }
+
+  bool set_one(size_t offset) {
+    auto i = offset / 8;
+    auto bit_i = offset % 8;
+    bits_size_ = std::max(bits_size_, offset + 1);
+    if (i >= bits_.size()) {
+      bits_.resize(std::max(i + 1, bits_.size() * 2));
+    }
+    auto mask = 1 << bit_i;
+    if ((bits_[i] & mask) == 0) {
+      bits_[i] |= mask;
+      count_++;
+      return true;
+    }
+    return false;
+  }
+
+  size_t ones_count() const {
+    return count_;
+  }
+
+  void set_raw(std::string bits) {
+    bits_ = std::move(bits);
+    bits_size_ = 0;
+    count_ = 0;
+    for (size_t n = size(), i = 0; i < n; i++) {
+      if (get(i)) {
+        count_++;
+        bits_size_ = i + 1;
+      }
+    }
+  }
+
+  size_t size() const {
+    return bits_.size() * 8;
+  }
+
+ private:
+  std::string bits_;
+  size_t bits_size_{0};
+  size_t count_{0};
+};
+}  // namespace td

storage/CMakeLists.txt

@@ -0,0 +1,55 @@
+cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
+
+if (NOT OPENSSL_FOUND)
+  find_package(OpenSSL REQUIRED)
+endif()
+
+
+set(STORAGE_SOURCE
+  LoadSpeed.cpp
+  MerkleTree.cpp
+  NodeActor.cpp
+  PeerActor.cpp
+  PeerState.cpp
+  Torrent.cpp
+  TorrentCreator.cpp
+  TorrentHeader.cpp
+  TorrentInfo.cpp
+  TorrentMeta.cpp
+
+  Bitset.h
+  LoadSpeed.h
+  MerkleTree.h
+  NodeActor.h
+  PartsHelper.h
+  PeerActor.h
+  PeerState.h
+  SharedState.h
+  Torrent.h
+  TorrentCreator.h
+  TorrentHeader.h
+  TorrentInfo.h
+  TorrentMeta.h
+)
+
+set(STORAGE_CLI_SOURCE
+  storage-cli.cpp
+)
+
+add_library(storage ${STORAGE_SOURCE})
+target_link_libraries(storage tdutils tdactor tddb ton_crypto tl_api ${JEMALLOC_LIBRARIES})
+target_include_directories(storage PUBLIC 
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}> 
+)
+
+add_executable(storage-cli ${STORAGE_CLI_SOURCE})
+target_link_libraries(storage-cli storage overlay tdutils tdactor adnl tl_api dht
+  rldp rldp2 catchain validatorsession full-node validator ton_validator validator
+  fift-lib memprof terminal ${JEMALLOC_LIBRARIES})
+
+set(STORAGE_TEST_SOURCE
+  ${CMAKE_CURRENT_SOURCE_DIR}/test/storage.cpp
+  PARENT_SCOPE
+)
+# Do not install it yet
+#install(TARGETS storage-cli RUNTIME DESTINATION bin)

storage/LoadSpeed.cpp

@@ -0,0 +1,33 @@
+#include "LoadSpeed.h"
+
+#include "td/utils/format.h"
+
+namespace ton {
+void LoadSpeed::add(td::size_t size, td::Timestamp now) {
+  total_size_ += size;
+  events_.push(Event{size, now});
+  update(now);
+}
+double LoadSpeed::speed(td::Timestamp now) const {
+  update(now);
+  return total_size_ / duration();
+}
+
+td::StringBuilder &operator<<(td::StringBuilder &sb, const LoadSpeed &speed) {
+  return sb << td::format::as_size(static_cast<td::uint64>(speed.speed())) << "/s";
+}
+
+void LoadSpeed::update(td::Timestamp now) const {
+  while (duration() > 60) {
+    total_size_ -= events_.front().size;
+    events_.pop();
+  }
+}
+double LoadSpeed::duration() const {
+  double res = 5;
+  if (events_.size() > 1) {
+    res = std::max(res, events_.back().at.at() - events_.front().at.at());
+  }
+  return res;
+}
+}  // namespace ton

storage/LoadSpeed.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "td/utils/StringBuilder.h"
+#include "td/utils/Time.h"
+#include "td/utils/VectorQueue.h"
+
+namespace ton {
+class LoadSpeed {
+ public:
+  void add(td::size_t size, td::Timestamp now);
+  double speed(td::Timestamp now = td::Timestamp::now()) const;
+  friend td::StringBuilder &operator<<(td::StringBuilder &sb, const LoadSpeed &speed);
+
+ private:
+  struct Event {
+    td::size_t size;
+    td::Timestamp at;
+  };
+  mutable td::VectorQueue<Event> events_;
+  mutable td::size_t total_size_{0};
+
+  double duration() const;
+  void update(td::Timestamp now) const;
+};
+}  // namespace ton

storage/MerkleTree.cpp

@@ -0,0 +1,342 @@
+#include "MerkleTree.h"
+
+#include "common/bitstring.h"
+#include "td/utils/UInt.h"
+
+#include "vm/cells/CellSlice.h"
+#include "vm/cells/MerkleProof.h"
+#include "vm/cellslice.h"
+#include "vm/excno.hpp"
+
+namespace ton {
+static td::Ref<vm::Cell> unpack_proof(td::Ref<vm::Cell> root) {
+  vm::CellSlice cs(vm::NoVm(), root);
+  CHECK(cs.special_type() == vm::Cell::SpecialType::MerkleProof);
+  return cs.fetch_ref();
+}
+
+td::uint32 MerkleTree::get_depth() const {
+  return log_n_;
+}
+td::Ref<vm::Cell> MerkleTree::get_root(size_t depth_limit) const {
+  if (depth_limit > log_n_ || root_proof_.is_null()) {
+    return root_proof_;
+  }
+
+  auto usage_tree = std::make_shared<vm::CellUsageTree>();
+  auto root_raw = vm::MerkleProof::virtualize(root_proof_, 1);
+  auto usage_cell = vm::UsageCell::create(root_raw, usage_tree->root_ptr());
+  do_gen_proof(std::move(usage_cell), unpack_proof(root_proof_), depth_limit);
+  auto res = vm::MerkleProof::generate(root_raw, usage_tree.get());
+  CHECK(res.not_null());
+  return res;
+}
+
+void MerkleTree::do_gen_proof(td::Ref<vm::Cell> node, td::Ref<vm::Cell> node_raw, size_t depth_limit) const {
+  if (depth_limit == 0) {
+    return;
+  }
+  // check if it is possible to load node without breaking virtualization
+  vm::CellSlice cs_raw(vm::NoVm(), std::move(node_raw));
+  if (cs_raw.is_special()) {
+    return;
+  }
+  vm::CellSlice cs(vm::NoVm(), std::move(node));
+  while (cs.have_refs()) {
+    do_gen_proof(cs.fetch_ref(), cs_raw.fetch_ref(), depth_limit - 1);
+  }
+}
+
+td::Bits256 MerkleTree::get_root_hash() const {
+  CHECK(root_hash_);
+  return root_hash_.value();
+}
+
+MerkleTree::MerkleTree(size_t chunks_count, td::Bits256 root_hash) {
+  init_begin(chunks_count);
+  root_hash_ = root_hash;
+  init_finish();
+}
+
+MerkleTree::MerkleTree(size_t chunks_count, td::Ref<vm::Cell> root_proof) {
+  init_begin(chunks_count);
+  root_hash_ = unpack_proof(root_proof)->get_hash(0).as_array();
+  root_proof_ = std::move(root_proof);
+  init_finish();
+}
+
+MerkleTree::MerkleTree(td::Span<Chunk> chunks) {
+  init_begin(chunks.size());
+
+  for (size_t i = 0; i < chunks.size(); i++) {
+    CHECK(chunks[i].index == i);
+    init_add_chunk(i, chunks[i].hash.as_slice());
+  }
+
+  init_finish();
+}
+
+void MerkleTree::init_begin(size_t chunks_count) {
+  log_n_ = 0;
+  while ((size_t(1) << log_n_) < chunks_count) {
+    log_n_++;
+  }
+  n_ = size_t(1) << log_n_;
+  total_blocks_ = chunks_count;
+  mark_.resize(n_ * 2);
+  proof_.resize(n_ * 2);
+
+  td::UInt256 null{};
+  auto cell = vm::CellBuilder().store_bytes(null.as_slice()).finalize();
+  for (auto i = chunks_count; i < n_; i++) {
+    proof_[i + n_] = cell;
+  }
+}
+
+void MerkleTree::init_add_chunk(size_t index, td::Slice hash) {
+  CHECK(index < total_blocks_);
+  CHECK(proof_[index + n_].is_null());
+  proof_[index + n_] = vm::CellBuilder().store_bytes(hash).finalize();
+}
+
+void MerkleTree::init_finish() {
+  for (size_t i = n_ - 1; i >= 1; i--) {
+    auto j = i * 2;
+    if (proof_[j].is_null()) {
+      continue;
+    }
+    if (i + 1 < n_ && proof_[i + 1].not_null() && proof_[j]->get_hash() == proof_[j + 2]->get_hash() &&
+        proof_[j + 1]->get_hash() == proof_[j + 3]->get_hash()) {
+      // minor optimization for same chunks
+      proof_[i] = proof_[i + 1];
+    } else {
+      proof_[i] = vm::CellBuilder().store_ref(proof_[j]).store_ref(proof_[j + 1]).finalize();
+    }
+  }
+  if (proof_[1].not_null()) {
+    init_proof();
+  }
+  CHECK(root_hash_);
+}
+
+void MerkleTree::remove_chunk(td::size_t index) {
+  CHECK(index < n_);
+  index += n_;
+  while (proof_[index].not_null()) {
+    proof_[index] = {};
+    index /= 2;
+  }
+}
+
+bool MerkleTree::has_chunk(td::size_t index) const {
+  CHECK(index < n_);
+  index += n_;
+  return proof_[index].not_null();
+}
+
+void MerkleTree::add_chunk(td::size_t index, td::Slice hash) {
+  CHECK(hash.size() == 32);
+  CHECK(index < n_);
+  index += n_;
+  auto cell = vm::CellBuilder().store_bytes(hash).finalize();
+  CHECK(proof_[index].is_null());
+  proof_[index] = std::move(cell);
+  mark_[index] = mark_id_;
+  for (index /= 2; index != 0; index /= 2) {
+    CHECK(proof_[index].is_null());
+    auto &left = proof_[index * 2];
+    auto &right = proof_[index * 2 + 1];
+    if (left.not_null() && right.not_null()) {
+      proof_[index] = vm::CellBuilder().store_ref(left).store_ref(right).finalize();
+      mark_[index] = mark_id_;
+    }
+  }
+}
+
+static td::Status do_validate(td::Ref<vm::Cell> ref, size_t depth) {
+  vm::CellSlice cs(vm::NoVm(), std::move(ref));
+  if (cs.is_special()) {
+    if (cs.special_type() != vm::Cell::SpecialType::PrunnedBranch) {
+      return td::Status::Error("Unexpected special cell");
+    }
+    return td::Status::OK();
+  }
+  if (depth == 0) {
+    if (cs.size() != 256) {
+      return td::Status::Error("List in proof must have 256 bits");
+    }
+    if (cs.size_refs() != 0) {
+      return td::Status::Error("List in proof must have zero refs");
+    }
+  } else {
+    if (cs.size() != 0) {
+      return td::Status::Error("Node in proof must have zero bits");
+    }
+    if (cs.size_refs() != 2) {
+      return td::Status::Error("Node in proof must have two refs");
+    }
+    TRY_STATUS(do_validate(cs.fetch_ref(), depth - 1));
+    TRY_STATUS(do_validate(cs.fetch_ref(), depth - 1));
+  }
+  return td::Status::OK();
+}
+
+td::Status MerkleTree::validate_proof(td::Ref<vm::Cell> new_root) {
+  // 1. depth <= log_n
+  // 2. each non special node has two refs and nothing else
+  // 3. each list contains only hash
+  // 4. all special nodes are merkle proofs
+  vm::CellSlice cs(vm::NoVm(), new_root);
+  if (cs.special_type() != vm::Cell::SpecialType::MerkleProof) {
+    return td::Status::Error("Proof must be a mekle proof cell");
+  }
+  auto root = cs.fetch_ref();
+  if (root_hash_ && root->get_hash(0).as_slice() != root_hash_.value().as_slice()) {
+    return td::Status::Error("Proof has invalid root hash");
+  }
+  return do_validate(std::move(root), log_n_);
+}
+
+td::Status MerkleTree::add_proof(td::Ref<vm::Cell> new_root) {
+  CHECK(root_proof_.not_null() || root_hash_);
+  TRY_STATUS(validate_proof(new_root));
+  if (root_proof_.not_null()) {
+    auto combined = vm::MerkleProof::combine_fast(root_proof_, std::move(new_root));
+    if (combined.is_null()) {
+      return td::Status::Error("Can't combine proofs");
+    }
+    root_proof_ = std::move(combined);
+  } else {
+    root_proof_ = std::move(new_root);
+  }
+  return td::Status::OK();
+}
+
+td::Status MerkleTree::validate_existing_chunk(const Chunk &chunk) {
+  vm::CellSlice cs(vm::NoVm(), proof_[chunk.index + n_]);
+  CHECK(cs.size() == chunk.hash.size());
+  if (cs.as_bitslice().compare(chunk.hash.cbits()) != 0) {
+    return td::Status::Error("Hash mismatch");
+  }
+  return td::Status::OK();
+}
+
+td::Status MerkleTree::try_add_chunks(td::Span<Chunk> chunks) {
+  td::Bitset bitmask;
+  add_chunks(chunks, bitmask);
+  for (size_t i = 0; i < chunks.size(); i++) {
+    if (!bitmask.get(i)) {
+      return td::Status::Error(PSLICE() << "Invalid chunk #" << chunks[i].index);
+    }
+  }
+  return td::Status::OK();
+}
+
+void MerkleTree::add_chunks(td::Span<Chunk> chunks, td::Bitset &bitmask) {
+  if (root_proof_.is_null()) {
+    return;
+  }
+
+  mark_id_++;
+  bitmask.reserve(chunks.size());
+  for (size_t i = 0; i < chunks.size(); i++) {
+    const auto &chunk = chunks[i];
+    if (has_chunk(chunk.index)) {
+      if (validate_existing_chunk(chunk).is_ok()) {
+        bitmask.set_one(i);
+      }
+      continue;
+    }
+    add_chunk(chunk.index, chunk.hash.as_slice());
+  }
+
+  root_proof_ = vm::CellBuilder::create_merkle_proof(merge(unpack_proof(root_proof_), 1));
+
+  for (size_t i = 0; i < chunks.size(); i++) {
+    const auto &chunk = chunks[i];
+    if (has_chunk(chunk.index) && mark_[chunk.index + n_] == mark_id_) {
+      bitmask.set_one(i);
+    }
+  }
+}
+
+td::Ref<vm::Cell> MerkleTree::merge(td::Ref<vm::Cell> root, size_t index) {
+  const auto &down = proof_[index];
+  if (down.not_null()) {
+    if (down->get_hash() != root->get_hash(0)) {
+      proof_[index] = {};
+    } else {
+      return down;
+    }
+  }
+
+  if (mark_[index] != mark_id_ || index >= n_) {
+    return root;
+  }
+
+  vm::CellSlice cs(vm::NoVm(), root);
+  if (cs.is_special()) {
+    cleanup_add(index);
+    return root;
+  }
+
+  CHECK(cs.size_refs() == 2);
+  vm::CellBuilder cb;
+  cb.store_bits(cs.fetch_bits(cs.size()));
+  auto left = merge(cs.fetch_ref(), index * 2);
+  auto right = merge(cs.fetch_ref(), index * 2 + 1);
+  cb.store_ref(std::move(left)).store_ref(std::move(right));
+  return cb.finalize();
+}
+
+void MerkleTree::cleanup_add(size_t index) {
+  if (mark_[index] != mark_id_) {
+    return;
+  }
+  proof_[index] = {};
+  if (index >= n_) {
+    return;
+  }
+  cleanup_add(index * 2);
+  cleanup_add(index * 2 + 1);
+}
+
+void MerkleTree::init_proof() {
+  CHECK(proof_[1].not_null());
+  td::Bits256 new_root_hash = proof_[1]->get_hash(0).as_array();
+  CHECK(!root_hash_ || root_hash_.value() == new_root_hash);
+  root_hash_ = new_root_hash;
+  root_proof_ = vm::CellBuilder::create_merkle_proof(proof_[1]);
+}
+
+td::Result<td::Ref<vm::Cell>> MerkleTree::gen_proof(size_t l, size_t r) {
+  if (root_proof_.is_null()) {
+    return td::Status::Error("got no proofs yet");
+  }
+  auto usage_tree = std::make_shared<vm::CellUsageTree>();
+  auto root_raw = vm::MerkleProof::virtualize(root_proof_, 1);
+  auto usage_cell = vm::UsageCell::create(root_raw, usage_tree->root_ptr());
+  TRY_STATUS(TRY_VM(do_gen_proof(std::move(usage_cell), 0, n_ - 1, l, r)));
+  auto res = vm::MerkleProof::generate(root_raw, usage_tree.get());
+  CHECK(res.not_null());
+  return res;
+}
+
+td::Status MerkleTree::do_gen_proof(td::Ref<vm::Cell> node, size_t il, size_t ir, size_t l, size_t r) const {
+  if (ir < l || il > r) {
+    return td::Status::OK();
+  }
+  if (l <= il && ir <= r) {
+    return td::Status::OK();
+  }
+  vm::CellSlice cs(vm::NoVm(), std::move(node));
+  if (cs.is_special()) {
+    return td::Status::Error("Can't generate a proof");
+  }
+  CHECK(cs.size_refs() == 2);
+  auto ic = (il + ir) / 2;
+  TRY_STATUS(do_gen_proof(cs.fetch_ref(), il, ic, l, r));
+  TRY_STATUS(do_gen_proof(cs.fetch_ref(), ic + 1, ir, l, r));
+  return td::Status::OK();
+}
+}  // namespace ton

storage/MerkleTree.h

@@ -0,0 +1,77 @@
+#pragma once
+
+#include "td/utils/optional.h"
+#include "td/utils/Slice.h"
+#include "vm/cells.h"
+
+#include "Bitset.h"
+
+namespace ton {
+// merkle_node$_ {n:#} left:^(ton::MerkleTree n) right:^(ton::MerkleTree n) = ton::MerkleTree (n + 1);
+// merkle_leaf$_ hash:bits256 = ton::MerkleTree 0;
+
+class MerkleTree {
+ public:
+  td::uint32 get_depth() const;
+  td::Ref<vm::Cell> get_root(size_t depth_limit = std::numeric_limits<size_t>::max()) const;
+  td::Bits256 get_root_hash() const;
+
+  MerkleTree(size_t chunks_count, td::Bits256 root_hash);
+  MerkleTree(size_t chunks_count, td::Ref<vm::Cell> root_proof);
+
+  struct Chunk {
+    td::size_t index{0};
+    td::Bits256 hash;
+  };
+
+  explicit MerkleTree(td::Span<Chunk> chunks);
+
+  MerkleTree() = default;
+  void init_begin(size_t chunks_count);
+  void init_add_chunk(td::size_t index, td::Slice hash);
+  void init_finish();
+
+  // merge external proof with an existing proof
+  td::Status add_proof(td::Ref<vm::Cell> new_root);
+  // generate proof for all chunks from l to r inclusive
+  td::Result<td::Ref<vm::Cell>> gen_proof(size_t l, size_t r);
+
+  // Trying to add and validate list of chunks simultaniously
+  td::Status try_add_chunks(td::Span<Chunk> chunks);
+
+  // Returns bitmask of successfully added chunks
+  // Intended to be used during validation of a torrent.
+  // We got arbitrary chunks read from disk, and we got an arbirary proof.
+  // Now we can say about some chunks that they are correct. This ia a general way
+  // to do this.
+  //
+  // NB: already added chunks are simply validated. One should be careful
+  // not to process them twice
+  void add_chunks(td::Span<Chunk> chunks, td::Bitset &bitmask);
+
+ private:
+  td::uint64 total_blocks_;
+  td::size_t n_;  // n = 2^log_n
+  td::uint32 log_n_;
+  td::size_t mark_id_{0};
+  std::vector<td::size_t> mark_;          // n_ * 2
+  std::vector<td::Ref<vm::Cell>> proof_;  // n_ * 2
+
+  td::optional<td::Bits256> root_hash_;
+  td::Ref<vm::Cell> root_proof_;
+
+  td::Status validate_proof(td::Ref<vm::Cell> new_root);
+  bool has_chunk(td::size_t index) const;
+  void remove_chunk(td::size_t index);
+
+  void add_chunk(td::size_t index, td::Slice hash);
+  void init_proof();
+
+  td::Ref<vm::Cell> merge(td::Ref<vm::Cell> root, size_t index);
+  void cleanup_add(size_t index);
+  td::Status do_gen_proof(td::Ref<vm::Cell> node, size_t il, size_t ir, size_t l, size_t r) const;
+  void do_gen_proof(td::Ref<vm::Cell> node, td::Ref<vm::Cell> node_raw, size_t depth_limit) const;
+  td::Status validate_existing_chunk(const Chunk &chunk);
+};
+
+}  // namespace ton

storage/NodeActor.cpp

@@ -0,0 +1,379 @@
+#include "NodeActor.h"
+
+#include "vm/boc.h"
+
+#include "td/utils/Enumerator.h"
+#include "td/utils/tests.h"
+
+namespace ton {
+NodeActor::NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback, bool should_download)
+    : self_id_(self_id)
+    , torrent_(std::move(torrent))
+    , callback_(std::move(callback))
+    , should_download_(should_download)
+    , parts_helper_(torrent.get_info().pieces_count()) {
+}
+
+void NodeActor::start_peer(PeerId peer_id, td::Promise<td::actor::ActorId<PeerActor>> promise) {
+  peers_[peer_id];
+  loop();
+  auto it = peers_.find(peer_id);
+  if (it == peers_.end() || it->second.actor.empty()) {
+    promise.set_error(td::Status::Error("Won't start peer now"));
+    return;
+  }
+  promise.set_value(it->second.actor.get());
+}
+
+void NodeActor::on_signal_from_peer(PeerId peer_id) {
+  loop_peer(peer_id, peers_[peer_id]);
+}
+
+void NodeActor::start_up() {
+  callback_->register_self(actor_id(this));
+  auto pieces_count = torrent_.get_info().pieces_count();
+  parts_.parts.resize(pieces_count);
+
+  auto header = torrent_.get_header_parts_range();
+  for (td::uint32 i = static_cast<td::uint32>(header.begin); i < header.end; i++) {
+    parts_helper_.set_part_priority(i, 255);
+  }
+  for (td::uint32 i = 0; i < pieces_count; i++) {
+    if (torrent_.is_piece_ready(i)) {
+      parts_helper_.on_self_part_ready(i);
+      parts_.parts[i].ready = true;
+    }
+  }
+  loop();
+}
+
+void NodeActor::loop_will_upload() {
+  if (peers_.empty()) {
+    return;
+  }
+
+  if (!will_upload_at_.is_in_past()) {
+    alarm_timestamp().relax(will_upload_at_);
+    return;
+  }
+
+  will_upload_at_ = td::Timestamp::in(5);
+  alarm_timestamp().relax(will_upload_at_);
+  std::vector<std::tuple<bool, bool, double, PeerId>> peers;
+  for (auto &it : peers_) {
+    auto state = it.second.state.lock();
+    bool needed = false;
+    if (state->peer_state_) {
+      needed = state->peer_state_.value().want_download;
+    }
+    peers.emplace_back(!needed, !state->node_state_.want_download, -state->download.speed(), it.first);
+  }
+  std::sort(peers.begin(), peers.end());
+
+  if (peers.size() > 5) {
+    std::swap(peers[4], peers[td::Random::fast(5, (int)peers.size() - 1)]);
+    peers.resize(5);
+  }
+
+  std::set<PeerId> peers_set;
+  for (auto id : peers) {
+    peers_set.insert(std::get<PeerId>(id));
+  }
+
+  for (auto &it : peers_) {
+    auto will_upload = peers_set.count(it.first) > 0;
+    auto state = it.second.state.lock();
+    if (state->node_state_.will_upload != will_upload) {
+      state->node_state_.will_upload = will_upload;
+      state->notify_peer();
+    }
+  }
+}
+
+void NodeActor::loop() {
+  loop_get_peers();
+  loop_start_stop_peers();
+  loop_queries();
+  loop_will_upload();
+
+  if (!ready_parts_.empty()) {
+    for (auto &it : peers_) {
+      auto state = it.second.state.lock();
+      state->node_ready_parts_.insert(state->node_ready_parts_.end(), ready_parts_.begin(), ready_parts_.end());
+      state->notify_peer();
+    }
+    ready_parts_.clear();
+  }
+
+  if (torrent_.is_completed() && !is_completed_) {
+    is_completed_ = true;
+    callback_->on_completed();
+  }
+}
+
+std::string NodeActor::get_stats_str() {
+  td::StringBuilder sb;
+  sb << "Node " << self_id_ << " " << torrent_.get_ready_parts_count() << "\t" << download_;
+  sb << "\toutq " << parts_.total_queries;
+  sb << "\n";
+  for (auto &it : peers_) {
+    auto state = it.second.state.lock();
+    sb << "\tPeer " << it.first;
+    sb << "\t" << parts_helper_.get_ready_parts(it.second.peer_token).ones_count();
+    sb << "\t" << state->download;
+    if (state->peer_state_) {
+      auto &peer_state = state->peer_state_.value();
+      sb << "\t  up:" << peer_state.will_upload;
+      sb << "\tdown:" << peer_state.want_download;
+      sb << "\tcnt:" << parts_helper_.get_want_download_count(it.second.peer_token);
+    }
+    sb << "\toutq:" << state->node_queries_.size();
+    sb << "\tinq:" << state->peer_queries_.size();
+    auto &node_state = state->node_state_;
+    sb << "\tNup:" << node_state.will_upload;
+    sb << "\tNdown:" << node_state.want_download;
+    sb << "\n";
+  }
+
+  auto o_n = torrent_.get_files_count();
+  if (o_n) {
+    // by default all parts priority == 1
+    auto n = o_n.unwrap();
+    file_priority_.resize(n, 1);
+    for (size_t i = 0; i < n; i++) {
+      auto size = torrent_.get_file_size(i);
+      auto ready_size = torrent_.get_file_ready_size(i);
+      sb << "#" << i << " " << torrent_.get_file_name(i) << "\t" << 100 * ready_size / size << "%%  "
+         << td::format::as_size(ready_size) << "/" << td::format::as_size(size) << "\t priority=" << file_priority_[i]
+         << "\n";
+    }
+  }
+  return sb.as_cslice().str();
+}
+
+void NodeActor::set_file_priority(size_t i, td::uint8 priority) {
+  auto o_files_count = torrent_.get_files_count();
+  if (!o_files_count) {
+    return;
+  }
+  auto files_count = o_files_count.unwrap();
+  if (file_priority_.size() != files_count) {
+    // by default all parts priority == 1
+    file_priority_.resize(files_count, 1);
+  }
+
+  if (i >= files_count) {
+    for (td::uint32 part_i = 0; part_i < torrent_.get_info().pieces_count(); part_i++) {
+      parts_helper_.set_part_priority(part_i, priority);
+    }
+    for (auto &p : file_priority_) {
+      p = priority;
+    }
+    return;
+  }
+  if (file_priority_[i] == priority) {
+    return;
+  }
+  file_priority_[i] = priority;
+  auto range = torrent_.get_file_parts_range(i);
+  td::uint32 begin = static_cast<td::uint32>(range.begin);
+  td::uint32 end = static_cast<td::uint32>(range.end);
+  for (td::uint32 i = begin; i < end; i++) {
+    if (i == begin || i + 1 == end) {
+      auto chunks = torrent_.chunks_by_piece(i);
+      td::uint8 max_priority = 0;
+      for (auto chunk_id : chunks) {
+        if (chunk_id == 0) {
+          max_priority = 255;
+        } else {
+          max_priority = td::max(max_priority, file_priority_[chunk_id - 1]);
+        }
+      }
+      parts_helper_.set_part_priority(i, max_priority);
+    } else {
+      parts_helper_.set_part_priority(i, priority);
+    }
+  }
+  yield();
+}
+
+void NodeActor::set_should_download(bool should_download) {
+  should_download_ = should_download;
+  yield();
+}
+
+void NodeActor::tear_down() {
+  callback_->on_closed(std::move(torrent_));
+}
+
+void NodeActor::loop_start_stop_peers() {
+  for (auto &it : peers_) {
+    auto &peer = it.second;
+    auto peer_id = it.first;
+
+    if (peer.notifier.empty()) {
+      peer.notifier = td::actor::create_actor<Notifier>("Notifier", actor_id(this), peer_id);
+    }
+
+    if (peer.actor.empty()) {
+      LOG(ERROR) << "Init Peer " << self_id_ << " -> " << peer_id;
+      auto state = peer.state.lock();
+      state->node = peer.notifier.get();
+      for (td::uint32 i = 0; i < parts_.parts.size(); i++) {
+        if (parts_.parts[i].ready) {
+          state->node_ready_parts_.push_back(i);
+        }
+      }
+      peer.peer_token = parts_helper_.register_peer(peer_id);
+      peer.actor = callback_->create_peer(self_id_, peer_id, peer.state);
+    }
+  }
+}
+
+void NodeActor::loop_queries() {
+  if (!should_download_) {
+    return;
+  }
+  for (auto &it : peers_) {
+    auto peer_token = it.second.peer_token;
+    auto state = it.second.state.lock();
+    if (!state->peer_state_) {
+      parts_helper_.set_peer_limit(peer_token, 0);
+      continue;
+    }
+    if (!state->peer_state_.value().will_upload) {
+      parts_helper_.set_peer_limit(peer_token, 0);
+      continue;
+    }
+    parts_helper_.set_peer_limit(peer_token,
+                                 td::narrow_cast<td::uint32>(MAX_PEER_TOTAL_QUERIES - state->node_queries_.size()));
+  }
+
+  auto parts = parts_helper_.get_rarest_parts(MAX_TOTAL_QUERIES);
+  for (auto &part : parts) {
+    auto it = peers_.find(part.peer_id);
+    CHECK(it != peers_.end());
+    auto state = it->second.state.lock();
+    CHECK(state->peer_state_);
+    CHECK(state->peer_state_.value().will_upload);
+    CHECK(state->node_queries_.size() < MAX_PEER_TOTAL_QUERIES);
+    auto part_id = part.part_id;
+    state->node_queries_[static_cast<td::uint32>(part_id)];
+    parts_helper_.lock_part(part_id);
+    parts_.total_queries++;
+    parts_.parts[part_id].query_to_peer = part.peer_id;
+    state->notify_peer();
+  }
+}
+
+void NodeActor::loop_get_peers() {
+  if (has_get_peers_) {
+    return;
+  }
+  if (next_get_peers_at_.is_in_past()) {
+    callback_->get_peers(promise_send_closure(td::actor::actor_id(this), &NodeActor::got_peers));
+    has_get_peers_ = true;
+    return;
+  }
+  alarm_timestamp().relax(next_get_peers_at_);
+}
+
+void NodeActor::got_peers(td::Result<std::vector<PeerId>> r_peers) {
+  if (r_peers.is_error()) {
+    next_get_peers_at_ = td::Timestamp::in(GET_PEER_RETRY_TIMEOUT);
+  } else {
+    auto peers = r_peers.move_as_ok();
+    for (auto &peer : peers) {
+      if (peer == self_id_) {
+        continue;
+      }
+      peers_[peer];
+    }
+    next_get_peers_at_ = td::Timestamp::in(GET_PEER_EACH);
+  }
+  has_get_peers_ = false;
+  loop();
+}
+
+void NodeActor::loop_peer(const PeerId &peer_id, Peer &peer) {
+  auto state = peer.state.lock();
+  CHECK(!state->peer.empty());
+
+  for (auto part_id : state->peer_ready_parts_) {
+    parts_helper_.on_peer_part_ready(peer.peer_token, part_id);
+  }
+  state->peer_ready_parts_.clear();
+
+  // Answer queries from peer
+  bool should_notify_peer = false;
+
+  auto want_download = parts_helper_.get_want_download_count(peer.peer_token) > 0;
+  if (state->node_state_.want_download != want_download) {
+    state->node_state_.want_download = want_download;
+    should_notify_peer = true;
+  }
+
+  for (auto it = state->peer_queries_.begin(); it != state->peer_queries_.end();) {
+    if (it->second) {
+      it++;
+    } else {
+      should_notify_peer = true;
+      it->second = [&]() -> td::Result<PeerState::Part> {
+        if (!state->node_state_.will_upload) {
+          return td::Status::Error("Won't upload");
+        }
+        TRY_RESULT(proof, torrent_.get_piece_proof(it->first));
+        TRY_RESULT(data, torrent_.get_piece_data(it->first));
+        PeerState::Part res;
+        TRY_RESULT(proof_serialized, vm::std_boc_serialize(std::move(proof)));
+        res.proof = std::move(proof_serialized);
+        res.data = td::BufferSlice(std::move(data));
+        return std::move(res);
+      }();
+    }
+  }
+
+  // Handle results from peer
+  for (auto it = state->node_queries_.begin(); it != state->node_queries_.end();) {
+    if (it->second) {
+      auto part_id = it->first;
+      auto r_unit = it->second.unwrap().move_fmap([&](PeerState::Part part) -> td::Result<td::Unit> {
+        TRY_RESULT(proof, vm::std_boc_deserialize(part.proof));
+        TRY_STATUS(torrent_.add_piece(part_id, part.data.as_slice(), std::move(proof)));
+        download_.add(part.data.size(), td::Timestamp::now());
+        return td::Unit();
+      });
+
+      parts_.parts[part_id].query_to_peer = {};
+      parts_.total_queries--;
+      it = state->node_queries_.erase(it);
+      parts_helper_.unlock_part(part_id);
+
+      if (r_unit.is_ok()) {
+        on_part_ready(part_id);
+      } else {
+        //LOG(ERROR) << "Failed " << part_id;
+      }
+    } else {
+      it++;
+    }
+  }
+
+  if (should_notify_peer) {
+    state->notify_peer();
+  }
+
+  yield();
+}
+
+void NodeActor::on_part_ready(PartId part_id) {
+  parts_helper_.on_self_part_ready(part_id);
+  CHECK(!parts_.parts[part_id].ready);
+  parts_.parts[part_id].ready = true;
+  for (auto &peer : peers_) {
+    // TODO: notify only peer want_download_count == 0
+    peer.second.state.unsafe()->notify_node();
+  }
+  ready_parts_.push_back(part_id);
+}
+}  // namespace ton

storage/NodeActor.h

@@ -0,0 +1,127 @@
+#pragma once
+
+#include "LoadSpeed.h"
+#include "PartsHelper.h"
+#include "PeerActor.h"
+#include "Torrent.h"
+
+#include "td/utils/Random.h"
+
+#include <map>
+
+namespace ton {
+class NodeActor : public td::actor::Actor {
+ public:
+  class Callback {
+   public:
+    virtual ~Callback() {
+    }
+    virtual td::actor::ActorOwn<PeerActor> create_peer(PeerId self_id, PeerId peer_id,
+                                                       td::SharedState<PeerState> state) = 0;
+    virtual void get_peers(td::Promise<std::vector<PeerId>> peers) = 0;
+    virtual void register_self(td::actor::ActorId<ton::NodeActor> self) = 0;
+
+    //TODO: proper callbacks
+    virtual void on_completed() = 0;
+    virtual void on_closed(ton::Torrent torrent) = 0;
+  };
+
+  NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback, bool should_download = true);
+  void start_peer(PeerId peer_id, td::Promise<td::actor::ActorId<PeerActor>> promise);
+
+  ton::Torrent *with_torrent() {
+    return &torrent_;
+  }
+  std::string get_stats_str();
+
+  void set_file_priority(size_t i, td::uint8 priority);
+  void set_should_download(bool should_download);
+
+ private:
+  PeerId self_id_;
+  ton::Torrent torrent_;
+  std::vector<td::uint8> file_priority_;
+  td::unique_ptr<Callback> callback_;
+  bool should_download_{false};
+
+  class Notifier : public td::actor::Actor {
+   public:
+    Notifier(td::actor::ActorId<NodeActor> node, PeerId peer_id) : node_(std::move(node)), peer_id_(peer_id) {
+    }
+
+    void wake_up() override {
+      send_closure(node_, &NodeActor::on_signal_from_peer, peer_id_);
+    }
+
+   private:
+    td::actor::ActorId<NodeActor> node_;
+    PeerId peer_id_;
+  };
+
+  struct Peer {
+    td::actor::ActorOwn<PeerActor> actor;
+    td::actor::ActorOwn<Notifier> notifier;
+    td::SharedState<PeerState> state;
+    PartsHelper::PeerToken peer_token;
+  };
+
+  std::map<PeerId, Peer> peers_;
+
+  struct QueryId {
+    PeerId peer;
+    PartId part;
+
+    auto key() const {
+      return std::tie(peer, part);
+    }
+    bool operator<(const QueryId &other) const {
+      return key() < other.key();
+    }
+  };
+
+  struct PartsSet {
+    struct Info {
+      td::optional<PeerId> query_to_peer;
+      bool ready{false};
+    };
+    size_t total_queries{0};
+    std::vector<Info> parts;
+  };
+
+  PartsSet parts_;
+  PartsHelper parts_helper_;
+  std::vector<PartId> ready_parts_;
+  LoadSpeed download_;
+
+  td::Timestamp next_get_peers_at_;
+  bool has_get_peers_{false};
+  static constexpr double GET_PEER_RETRY_TIMEOUT = 5;
+  static constexpr double GET_PEER_EACH = 5;
+
+  bool is_completed_{false};
+
+  td::Timestamp will_upload_at_;
+
+  void on_signal_from_peer(PeerId peer_id);
+
+  void start_up() override;
+
+  void loop() override;
+
+  void tear_down() override;
+
+  void loop_start_stop_peers();
+
+  static constexpr size_t MAX_TOTAL_QUERIES = 20;
+  static constexpr size_t MAX_PEER_TOTAL_QUERIES = 5;
+  void loop_queries();
+  bool try_send_query();
+  bool try_send_part(PartId part_id);
+  void loop_get_peers();
+  void got_peers(td::Result<std::vector<PeerId>> r_peers);
+  void loop_peer(const PeerId &peer_id, Peer &peer);
+  void on_part_ready(PartId part_id);
+
+  void loop_will_upload();
+};
+}  // namespace ton

storage/PartsHelper.h

@@ -0,0 +1,259 @@
+#include "PeerState.h"
+#include "Bitset.h"
+
+#include "td/utils/Random.h"
+#include "td/utils/Status.h"
+
+namespace ton {
+struct PartsHelper {
+ public:
+  PartsHelper(size_t parts_count) : parts_(parts_count), peers_(64) {
+    peers_[0].is_valid = true;
+  }
+  using PartId = size_t;
+  using PeerToken = size_t;
+
+  PeerToken register_self() {
+    return self_token_;
+  }
+  PeerToken register_peer(PeerId peer_id) {
+    PeerToken new_peer_token{free_peer_tokens_.empty() ? next_peer_token_ : free_peer_tokens_.back()};
+    auto it = peer_id_to_token_.emplace(peer_id, new_peer_token);
+    if (it.second) {
+      if (free_peer_tokens_.empty()) {
+        next_peer_token_++;
+        peers_.resize(next_peer_token_);
+      } else {
+        free_peer_tokens_.pop_back();
+      }
+      auto peer = get_peer(new_peer_token, true);
+      peer->is_valid = true;
+      peer->peer_id = peer_id;
+      peer->want_download_count = 0;
+    }
+    return it.first->second;
+  }
+  void forget_peer(PeerToken peer_token) {
+    CHECK(peer_token != self_token_);
+    free_peer_tokens_.push_back(peer_token);
+    auto *peer = get_peer(peer_token);
+    peer_id_to_token_.erase(get_peer(peer_token)->peer_id);
+    *peer = Peer{};
+    peer->rnd = td::Random::fast_uint32();
+  }
+
+  void set_peer_limit(PeerToken peer, td::uint32 limit) {
+    get_peer(peer)->limit = limit;
+  }
+
+  void on_peer_part_ready(PeerToken peer_token, PartId part_id) {
+    auto peer = get_peer(peer_token);
+    if (!peer->ready_parts.set_one(part_id)) {
+      return;
+    }
+    auto part = get_part(part_id);
+    if (part->is_ready) {
+      return;
+    }
+    peer->want_download_count++;
+    if (!part->rnd) {
+      part->rnd = td::Random::fast_uint32();
+    }
+    change_key(part_id, part->rnd, part->peers_count, part->peers_count + 1, part->priority, part->priority);
+    part->peers_count++;
+  }
+
+  void lock_part(PartId part_id) {
+    auto *part = get_part(part_id);
+    CHECK(!part->is_locked);
+    part->is_locked = true;
+  }
+  void unlock_part(PartId part_id) {
+    auto *part = get_part(part_id);
+    CHECK(part->is_locked);
+    part->is_locked = false;
+  }
+
+  void set_part_priority(PartId part_id, td::uint8 priority) {
+    auto *part = get_part(part_id);
+    if (part->is_ready) {
+      return;
+    }
+    change_key(part_id, part->rnd, part->peers_count, part->peers_count, part->priority, priority);
+    part->priority = priority;
+  }
+
+  td::uint8 get_part_priority(PartId part_id) {
+    auto *part = get_part(part_id);
+    return part->priority;
+  }
+
+  void on_self_part_ready(PartId part_id) {
+    auto peer = get_peer(self_token_);
+    if (!peer->ready_parts.set_one(part_id)) {
+      return;
+    }
+    auto part = get_part(part_id);
+    CHECK(!part->is_ready);
+    part->is_ready = true;
+    for (auto &peer : peers_) {
+      if (peer.ready_parts.get(part_id)) {
+        peer.want_download_count--;
+      }
+    }
+    change_key(part_id, part->rnd, part->peers_count, 0, part->priority, part->priority);
+  }
+
+  struct RarePart {
+    PartId part_id;
+    PeerId peer_id;
+  };
+  std::vector<RarePart> get_rarest_parts(size_t max_count) {
+    struct It {
+      std::set<Peer::Key>::iterator begin, end;
+      PeerId peer_id;
+      td::uint32 limit{0};
+      bool empty() const {
+        return begin == end || limit == 0;
+      }
+      auto key() const {
+        return std::tie(*begin, peer_id);
+      }
+      bool operator<(const It &other) const {
+        return key() < other.key();
+      }
+    };
+    std::set<It> its;
+
+    for (auto &peer : peers_) {
+      if (!peer.is_valid || peer.limit == 0) {
+        continue;
+      }
+      It it;
+      it.begin = peer.rarest_parts.begin();
+      it.end = peer.rarest_parts.end();
+      it.peer_id = peer.peer_id;
+      it.limit = peer.limit;
+      if (it.empty()) {
+        continue;
+      }
+      its.insert(it);
+    }
+
+    std::vector<RarePart> res;
+    while (res.size() < max_count && !its.empty()) {
+      auto it = *its.begin();
+      its.erase(its.begin());
+      auto part_id = it.begin->part_id;
+      if ((res.empty() || res.back().part_id != part_id) && !get_part(part_id)->is_locked) {
+        res.push_back({part_id, it.peer_id});
+        CHECK(get_peer(register_peer(it.peer_id))->ready_parts.get(part_id));
+        it.limit--;
+      }
+      it.begin++;
+      if (it.empty()) {
+        continue;
+      }
+      its.insert(it);
+    }
+    return res;
+  }
+
+  td::uint32 get_want_download_count(PeerToken peer_token) {
+    return get_peer(peer_token, false)->want_download_count;
+  }
+  const td::Bitset &get_ready_parts(PeerToken peer_token) {
+    return get_peer(peer_token, false)->ready_parts;
+  }
+
+ private:
+  PeerToken self_token_{0};
+  size_t parts_count_;
+  struct Part {
+    bool is_locked{false};
+    bool is_ready{false};
+    td::uint8 priority{1};
+    td::uint32 rnd{0};
+    td::uint32 peers_count{0};  // invalid for ready parts
+  };
+  struct Peer {
+    PeerId peer_id{0};
+    bool is_valid{false};
+    td::uint32 rnd{0};
+    td::uint32 limit{0};
+    td::Bitset ready_parts;
+    // sum_i (peer.ready_parts[i] && !node.ready_parts[i])
+    td::uint32 want_download_count{0};
+
+    // peers_count - count of peers which has this part
+    // key_count = !is_ready * peers_count;
+    struct Key {
+      td::uint8 priority{0};
+      td::uint32 count{0};
+      PartId part_id{0};
+      td::uint32 rnd{0};
+      auto key() const {
+        return std::make_tuple(255 - priority, count, rnd, part_id);
+      }
+      bool operator<(const Key &other) const {
+        return key() < other.key();
+      }
+    };
+    std::set<Key> rarest_parts;  // TODO: use vector instead of set
+  };
+
+  std::vector<Part> parts_;
+  std::vector<Peer> peers_;
+  td::uint32 next_peer_token_{1};
+  std::map<PeerId, PeerToken> peer_id_to_token_;
+  std::vector<PeerToken> free_peer_tokens_;
+
+  Part *get_part(PartId part_id) {
+    CHECK(part_id < parts_.size());
+    return &parts_[part_id];
+  }
+  Peer *get_peer(PeerToken peer_token, bool can_be_uninited = false) {
+    CHECK(peer_token < peers_.size());
+    auto res = &peers_[peer_token];
+    CHECK(res->is_valid || can_be_uninited);
+    return res;
+  }
+
+  void change_key(PartId part_id, td::uint32 rnd, td::uint32 from_count, td::uint32 to_count, td::uint8 from_priority,
+                  td::uint8 to_priority) {
+    if (from_count == 0 && to_count == 0) {
+      return;
+    }
+    if (from_count == to_count && from_priority == to_priority) {
+      return;
+    }
+    for (auto &peer : peers_) {
+      if (!peer.is_valid) {
+        continue;
+      }
+      if (peer.peer_id == 0) {
+        continue;
+      }
+      if (!peer.ready_parts.get(part_id)) {
+        continue;
+      }
+      //TODO: xor is not a perfect solution as it keeps a lot order between part_ids
+      Peer::Key key;
+      key.part_id = part_id;
+      key.rnd = rnd ^ peer.rnd;
+
+      if (from_count != 0 && from_priority != 0) {
+        key.count = from_count;
+        key.priority = from_priority;
+        peer.rarest_parts.erase(key);
+      }
+
+      if (to_count != 0 && to_priority != 0) {
+        key.count = to_count;
+        key.priority = to_priority;
+        peer.rarest_parts.insert(key);
+      }
+    }
+  }
+};
+}  // namespace ton

storage/PeerActor.cpp

@@ -0,0 +1,400 @@
+#include "PeerActor.h"
+#include "auto/tl/ton_api.hpp"
+
+#include "tl-utils/tl-utils.hpp"
+
+#include "td/utils/overloaded.h"
+#include "td/utils/Random.h"
+
+namespace ton {
+
+PeerState::State from_ton_api(const ton::ton_api::storage_state &state) {
+  PeerState::State res;
+  res.want_download = state.want_download_;
+  res.will_upload = state.will_upload_;
+  return res;
+}
+
+ton::ton_api::object_ptr<ton::ton_api::storage_state> to_ton_api(const PeerState::State &state) {
+  return ton::ton_api::make_object<ton::ton_api::storage_state>(state.will_upload, state.want_download);
+}
+
+PeerActor::PeerActor(td::unique_ptr<Callback> callback, td::SharedState<PeerState> state)
+    : callback_(std::move(callback)), state_(std::move(state)) {
+  CHECK(callback_);
+}
+
+template <class T, class... ArgsT>
+td::uint64 PeerActor::create_and_send_query(ArgsT &&... args) {
+  return send_query(ton::create_serialize_tl_object<T>(std::forward<ArgsT>(args)...));
+}
+
+td::uint64 PeerActor::send_query(td::BufferSlice query) {
+  auto query_id = next_query_id_++;
+  //LOG(ERROR) << "send_query " << to_string(ton::fetch_tl_object<ton::ton_api::Function>(std::move(query), true).ok());
+  callback_->send_query(query_id, std::move(query));
+  return query_id;
+}
+
+void PeerActor::schedule_loop() {
+  yield();
+}
+
+void PeerActor::notify_node() {
+  need_notify_node_ = true;
+}
+
+void PeerActor::execute_query(td::BufferSlice query, td::Promise<td::BufferSlice> promise) {
+  TRY_RESULT_PROMISE(promise, f, ton::fetch_tl_object<ton::ton_api::Function>(std::move(query), true));
+  //LOG(ERROR) << "execute_query " << to_string(f);
+  ton::ton_api::downcast_call(
+      *f, td::overloaded(
+              [&](ton::ton_api::storage_ping &ping) {
+                execute_ping(static_cast<td::uint64>(ping.session_id_), std::move(promise));
+              },
+              [&](ton::ton_api::storage_addUpdate &add_update) { execute_add_update(add_update, std::move(promise)); },
+              [&](ton::ton_api::storage_getPiece &get_piece) { execute_get_piece(get_piece, std::move(promise)); },
+              [&](auto &other) { promise.set_error(td::Status::Error("Unknown function")); }));
+  schedule_loop();
+}
+
+void PeerActor::on_ping_result(td::Result<td::BufferSlice> r_answer) {
+  ping_query_id_ = {};
+  if (r_answer.is_ok()) {
+    on_pong();
+  }
+}
+
+void PeerActor::on_pong() {
+  wait_pong_till_ = td::Timestamp::in(4);
+  state_.lock()->peer_online_ = true;
+  notify_node();
+}
+
+void PeerActor::on_update_result(td::Result<td::BufferSlice> r_answer) {
+  update_query_id_ = {};
+  if (r_answer.is_ok()) {
+    peer_is_inited_ = true;
+    have_pieces_list_.clear();
+  }
+}
+
+void PeerActor::on_get_piece_result(PartId piece_id, td::Result<td::BufferSlice> r_answer) {
+  auto state = state_.lock();
+  auto it = state->node_queries_.find(piece_id);
+  if (it == state->node_queries_.end()) {
+    LOG(ERROR) << "???";
+    return;
+  }
+  //TODO: handle errors ???
+  it->second = [&]() -> td::Result<PeerState::Part> {
+    TRY_RESULT(slice, std::move(r_answer));
+    TRY_RESULT(piece, ton::fetch_result<ton::ton_api::storage_getPiece>(slice.as_slice()));
+    PeerState::Part res;
+    res.data = std::move(piece->data_);
+    res.proof = std::move(piece->proof_);
+    return std::move(res);
+  }();
+  notify_node();
+}
+
+void PeerActor::on_update_state_result(td::Result<td::BufferSlice> r_answer) {
+  if (r_answer.is_error()) {
+    update_state_query_.query_id = {};
+  }
+}
+
+void PeerActor::on_query_result(td::uint64 query_id, td::Result<td::BufferSlice> r_answer) {
+  if (r_answer.is_ok()) {
+    on_pong();
+    state_.lock()->download.add(r_answer.ok().size(), td::Timestamp::now());
+  }
+  if (ping_query_id_ && ping_query_id_.value() == query_id) {
+    on_ping_result(std::move(r_answer));
+  } else if (update_query_id_ && update_query_id_.value() == query_id) {
+    on_update_result(std::move(r_answer));
+  } else if (update_state_query_.query_id && update_state_query_.query_id.value() == query_id) {
+    on_update_state_result(std::move(r_answer));
+  } else {
+    for (auto &query_it : node_get_piece_) {
+      if (query_it.second.query_id && query_it.second.query_id.value() == query_id) {
+        on_get_piece_result(query_it.first, std::move(r_answer));
+        query_it.second.query_id = {};
+      }
+    }
+  }
+
+  schedule_loop();
+}
+
+void PeerActor::start_up() {
+  callback_->register_self(actor_id(this));
+
+  node_session_id_ = td::Random::secure_uint64();
+
+  auto state = state_.lock();
+  state->peer = actor_id(this);
+
+  notify_node();
+  schedule_loop();
+}
+
+void PeerActor::loop() {
+  loop_ping();
+  loop_pong();
+
+  loop_update_init();
+  loop_update_state();
+  loop_update_pieces();
+
+  loop_node_get_piece();
+  loop_peer_get_piece();
+
+  loop_notify_node();
+}
+
+void PeerActor::loop_pong() {
+  if (wait_pong_till_ && wait_pong_till_.is_in_past()) {
+    wait_pong_till_ = {};
+    LOG(INFO) << "Disconnected";
+    state_.lock()->peer_online_ = false;
+    notify_node();
+  }
+  alarm_timestamp().relax(wait_pong_till_);
+}
+
+void PeerActor::loop_ping() {
+  if (ping_query_id_) {
+    return;
+  }
+  if (!next_ping_at_.is_in_past()) {
+    alarm_timestamp().relax(next_ping_at_);
+    return;
+  }
+
+  next_ping_at_ = td::Timestamp::in(2);
+  alarm_timestamp().relax(next_ping_at_);
+  ping_query_id_ = create_and_send_query<ton::ton_api::storage_ping>(node_session_id_);
+}
+
+td::BufferSlice PeerActor::create_update_query(ton::tl_object_ptr<ton::ton_api::storage_Update> update) {
+  auto session_id = static_cast<td::int64>(peer_session_id_.value());
+  auto seqno = static_cast<td::int32>(++node_seqno_);
+  return ton::create_serialize_tl_object<ton::ton_api::storage_addUpdate>(session_id, seqno, std::move(update));
+}
+
+void PeerActor::loop_update_init() {
+  if (!peer_session_id_) {
+    return;
+  }
+  if (update_query_id_) {
+    return;
+  }
+  if (peer_is_inited_) {
+    return;
+  }
+
+  update_have_pieces();
+  have_pieces_list_.clear();
+
+  auto state = state_.lock();
+  auto query = create_update_query(ton::create_tl_object<ton::ton_api::storage_updateInit>(
+      td::BufferSlice(have_pieces_.as_slice()), to_ton_api(state->node_state_)));
+
+  // take care about update_state_query initial state
+  update_state_query_.state = state->node_state_;
+  update_state_query_.query_id = 0;
+
+  update_query_id_ = send_query(std::move(query));
+}
+
+void PeerActor::loop_update_state() {
+  if (!peer_is_inited_) {
+    return;
+  }
+
+  auto state = state_.lock();
+  if (!(update_state_query_.state == state->node_state_)) {
+    update_state_query_.state = state->node_state_;
+    update_state_query_.query_id = {};
+  }
+
+  if (update_state_query_.query_id) {
+    return;
+  }
+
+  auto query = create_update_query(
+      ton::create_tl_object<ton::ton_api::storage_updateState>(to_ton_api(update_state_query_.state)));
+  update_state_query_.query_id = send_query(std::move(query));
+}
+
+void PeerActor::update_have_pieces() {
+  auto state = state_.lock();
+  have_pieces_list_.insert(have_pieces_list_.end(), state->node_ready_parts_.begin(), state->node_ready_parts_.end());
+  for (auto piece_id : state->node_ready_parts_) {
+    have_pieces_.set_one(piece_id);
+  }
+  state->node_ready_parts_.clear();
+}
+
+void PeerActor::loop_update_pieces() {
+  if (update_query_id_) {
+    return;
+  }
+
+  if (!peer_is_inited_) {
+    return;
+  }
+
+  update_have_pieces();
+
+  if (!have_pieces_list_.empty()) {
+    auto query = create_update_query(ton::create_tl_object<ton::ton_api::storage_updateHavePieces>(
+        td::transform(have_pieces_list_, [](auto x) { return static_cast<td::int32>(x); })));
+    update_query_id_ = send_query(std::move(query));
+  }
+}
+
+void PeerActor::loop_node_get_piece() {
+  auto state = state_.lock();
+
+  for (auto it = node_get_piece_.begin(); it != node_get_piece_.end();) {
+    auto other_it = state->node_queries_.find(it->first);
+    if (other_it == state->node_queries_.end() || other_it->second) {
+      it = node_get_piece_.erase(it);
+    } else {
+      it++;
+    }
+  }
+
+  for (auto &query_it : state->node_queries_) {
+    if (query_it.second) {
+      continue;
+    }
+    node_get_piece_.emplace(query_it.first, NodePieceQuery{});
+  }
+
+  for (auto &query_it : node_get_piece_) {
+    if (query_it.second.query_id) {
+      continue;
+    }
+
+    query_it.second.query_id =
+        create_and_send_query<ton::ton_api::storage_getPiece>(static_cast<td::int32>(query_it.first));
+  }
+}
+
+void PeerActor::loop_peer_get_piece() {
+  auto state = state_.lock();
+
+  // process answers
+  for (auto &it : state->peer_queries_) {
+    if (!it.second) {
+      continue;
+    }
+    auto promise_it = peer_get_piece_.find(it.first);
+    if (promise_it == peer_get_piece_.end()) {
+      continue;
+    }
+    promise_it->second.promise.set_result(it.second.unwrap().move_map([](PeerState::Part part) {
+      return ton::create_serialize_tl_object<ton::ton_api::storage_piece>(std::move(part.proof), std::move(part.data));
+    }));
+    peer_get_piece_.erase(promise_it);
+  }
+
+  // erase unneeded queries
+  for (auto it = state->peer_queries_.begin(); it != state->peer_queries_.end();) {
+    if (peer_get_piece_.count(it->first) == 0) {
+      it = state->peer_queries_.erase(it);
+      notify_node();
+    } else {
+      it++;
+    }
+  }
+
+  // create queries
+  for (auto &query_it : peer_get_piece_) {
+    auto res = state->peer_queries_.emplace(query_it.first, td::optional<td::Result<PeerState::Part>>());
+    if (res.second) {
+      notify_node();
+    }
+  }
+}
+
+void PeerActor::loop_notify_node() {
+  if (!need_notify_node_) {
+    return;
+  }
+  need_notify_node_ = false;
+  state_.lock()->notify_node();
+}
+
+void PeerActor::execute_ping(td::uint64 session_id, td::Promise<td::BufferSlice> promise) {
+  if (!peer_session_id_ || peer_session_id_.value() != session_id) {
+    peer_session_id_ = session_id;
+    peer_is_inited_ = false;
+
+    update_query_id_ = {};
+    update_state_query_.query_id = {};
+  }
+
+  promise.set_value(ton::create_serialize_tl_object<ton::ton_api::storage_pong>());
+}
+
+void PeerActor::execute_add_update(ton::ton_api::storage_addUpdate &add_update, td::Promise<td::BufferSlice> promise) {
+  auto session_id = static_cast<td::uint64>(add_update.session_id_);
+  if (session_id != node_session_id_) {
+    promise.set_error(td::Status::Error(404, "INVALID_SESSION"));
+    return;
+  }
+
+  promise.set_value(ton::create_serialize_tl_object<ton::ton_api::storage_ok>());
+
+  auto state = state_.lock();
+  auto add_piece = [&](PartId id) {
+    if (!peer_have_pieces_.get(id)) {
+      peer_have_pieces_.set_one(id);
+      state->peer_ready_parts_.push_back(id);
+      notify_node();
+    }
+  };
+
+  auto seqno = static_cast<td::uint32>(add_update.seqno_);
+  auto update_peer_state = [&](PeerState::State peer_state) {
+    if (peer_seqno_ >= seqno) {
+      return;
+    }
+    if (state->peer_state_ && state->peer_state_.value() == peer_state) {
+      return;
+    }
+    peer_seqno_ = seqno;
+    state->peer_state_ = peer_state;
+    notify_node();
+  };
+
+  //LOG(ERROR) << "Got " << to_string(add_update);
+  downcast_call(*add_update.update_,
+                td::overloaded(
+                    [&](ton::ton_api::storage_updateHavePieces &have_pieces) {
+                      for (auto id : have_pieces.piece_id_) {
+                        add_piece(id);
+                      }
+                    },
+                    [&](ton::ton_api::storage_updateState &state) { update_peer_state(from_ton_api(*state.state_)); },
+                    [&](ton::ton_api::storage_updateInit &init) {
+                      update_peer_state(from_ton_api(*init.state_));
+                      td::Bitset new_bitset;
+                      new_bitset.set_raw(init.have_pieces_.as_slice().str());
+                      for (auto size = new_bitset.size(), i = size_t(0); i < size; i++) {
+                        if (new_bitset.get(i)) {
+                          add_piece(static_cast<PartId>(i));
+                        }
+                      }
+                    }));
+}
+
+void PeerActor::execute_get_piece(ton::ton_api::storage_getPiece &get_piece, td::Promise<td::BufferSlice> promise) {
+  PartId piece_id = get_piece.piece_id_;
+  peer_get_piece_[piece_id] = {std::move(promise)};
+}
+}  // namespace ton

storage/PeerActor.h

@@ -0,0 +1,112 @@
+#pragma once
+
+#include "Bitset.h"
+#include "PeerState.h"
+#include "SharedState.h"
+
+#include "td/utils/optional.h"
+
+#include "auto/tl/ton_api.h"
+
+namespace ton {
+class PeerActor : public td::actor::Actor {
+ public:
+  class Callback {
+   public:
+    virtual ~Callback() {
+    }
+    virtual void register_self(td::actor::ActorId<PeerActor> self) = 0;
+    virtual void send_query(td::uint64 query_id, td::BufferSlice query) = 0;
+  };
+
+  PeerActor(td::unique_ptr<Callback> callback, td::SharedState<PeerState> state);
+
+  void execute_query(td::BufferSlice query, td::Promise<td::BufferSlice> promise);
+  void on_query_result(td::uint64 query_id, td::Result<td::BufferSlice> r_answer);
+
+ private:
+  td::unique_ptr<Callback> callback_;
+  td::SharedState<PeerState> state_;
+  bool need_notify_node_{false};
+
+  td::uint64 next_query_id_{0};
+
+  // ping
+  td::Timestamp next_ping_at_;
+  td::optional<td::uint64> ping_query_id_;
+  td::Timestamp wait_pong_till_;
+
+  // startSession
+  td::uint64 node_session_id_;
+  td::Bitset peer_have_pieces_;
+
+  // update
+  td::optional<td::uint64> peer_session_id_;
+  td::optional<td::uint64> update_query_id_;
+  bool peer_is_inited_{false};
+  td::uint32 node_seqno_{0};
+  td::Bitset have_pieces_;
+  std::vector<PartId> have_pieces_list_;
+  td::uint32 peer_seqno_{0};
+
+  // update state
+  struct UpdateState {
+    td::optional<td::uint64> query_id;
+    PeerState::State state;
+  };
+  UpdateState update_state_query_;
+
+  // getPiece
+  struct NodePieceQuery {
+    td::optional<td::uint64> query_id;
+  };
+  std::map<PartId, NodePieceQuery> node_get_piece_;
+
+  struct PeerPieceQuery {
+    td::Promise<td::BufferSlice> promise;
+  };
+  std::map<PartId, PeerPieceQuery> peer_get_piece_;
+
+  void start_up() override;
+
+  void loop() override;
+
+  void loop_notify_node();
+
+  void loop_pong();
+  void execute_ping(td::uint64 session_id, td::Promise<td::BufferSlice> promise);
+  void on_ping_result(td::Result<td::BufferSlice> r_answer);
+  void on_pong();
+
+  void loop_ping();
+
+  void loop_update_init();
+  void loop_update_pieces();
+  void update_have_pieces();
+
+  void loop_update_state();
+
+  td::BufferSlice create_update_query(ton::tl_object_ptr<ton::ton_api::storage_Update> update);
+
+  void loop_node_get_piece();
+
+  void loop_peer_get_piece();
+
+  void execute_add_update(ton::ton_api::storage_addUpdate &add_update, td::Promise<td::BufferSlice> promise);
+
+  void execute_get_piece(ton::ton_api::storage_getPiece &get_piece, td::Promise<td::BufferSlice> promise);
+
+  void on_update_result(td::Result<td::BufferSlice> r_answer);
+
+  void on_get_piece_result(PartId piece_id, td::Result<td::BufferSlice> r_answer);
+
+  void on_update_state_result(td::Result<td::BufferSlice> r_answer);
+
+  template <class T, class... ArgsT>
+  td::uint64 create_and_send_query(ArgsT &&... args);
+  td::uint64 send_query(td::BufferSlice query);
+
+  void schedule_loop();
+  void notify_node();
+};
+}  // namespace ton

storage/PeerState.cpp

@@ -0,0 +1,16 @@
+#include "PeerState.h"
+namespace ton {
+void PeerState::notify_node() {
+  if (node.empty()) {
+    return;
+  }
+  td::actor::send_signals_later(node, td::actor::ActorSignals::wakeup());
+}
+
+void PeerState::notify_peer() {
+  if (peer.empty()) {
+    return;
+  }
+  td::actor::send_signals_later(peer, td::actor::ActorSignals::wakeup());
+}
+}  // namespace ton

storage/PeerState.h

@@ -0,0 +1,55 @@
+#pragma once
+#include "td/utils/buffer.h"
+#include "td/utils/common.h"
+#include "td/utils/optional.h"
+
+#include "td/actor/actor.h"
+
+#include <map>
+
+#include "LoadSpeed.h"
+
+namespace ton {
+using PeerId = td::uint64;
+using PartId = td::uint32;
+
+struct PeerState {
+  struct State {
+    bool will_upload{false};
+    bool want_download{false};
+    auto key() const {
+      return std::tie(will_upload, want_download);
+    }
+    bool operator==(const State &other) const {
+      return key() == other.key();
+    }
+  };
+  State node_state_;
+  td::optional<State> peer_state_;
+  bool peer_online_{false};
+
+  struct Part {
+    td::BufferSlice proof;
+    td::BufferSlice data;
+  };
+  std::map<PartId, td::optional<td::Result<Part>>> node_queries_;
+  std::map<PartId, td::optional<td::Result<Part>>> peer_queries_;
+
+  // Peer -> Node
+  // update are added to this vector, so reader will be able to process all changes
+  std::vector<PartId> peer_ready_parts_;
+
+  // Node -> Peer
+  // writer writes all new parts to this vector. This state will be eventually synchornized with a peer
+  std::vector<PartId> node_ready_parts_;
+
+  td::actor::ActorId<> node;
+  td::actor::ActorId<> peer;
+
+  LoadSpeed upload;
+  LoadSpeed download;
+
+  void notify_node();
+  void notify_peer();
+};
+}  // namespace ton

storage/SharedState.h

@@ -0,0 +1,48 @@
+#pragma once
+
+#include <atomic>
+#include "td/utils/MovableValue.h"
+#include <memory>
+
+namespace td {
+template <class T>
+class SharedState {
+ public:
+  friend class Guard;
+  class Guard {
+   public:
+    Guard(Guard &&) = default;
+    Guard(SharedState<T> *self) : self(self) {
+      CHECK(!self->data_->is_locked.exchange(true));
+    }
+    ~Guard() {
+      if (self.get()) {
+        CHECK(self.get()->data_->is_locked.exchange(false));
+      }
+    }
+    T *get() {
+      return &self.get()->data_->data;
+    }
+    T *operator->() {
+      return get();
+    }
+
+   private:
+    td::MovableValue<SharedState<T> *> self;
+  };
+
+  auto lock() {
+    return Guard{this};
+  }
+  auto unsafe() {
+    return &data_->data;
+  }
+
+ private:
+  struct Data {
+    std::atomic<bool> is_locked{};
+    T data;
+  };
+  std::shared_ptr<Data> data_{std::make_shared<Data>()};
+};
+}  // namespace td

storage/Torrent.cpp

@@ -0,0 +1,425 @@
+#include "Torrent.h"
+
+#include "td/utils/Status.h"
+#include "td/utils/crypto.h"
+#include "td/utils/port/Stat.h"
+#include "td/utils/tl_helpers.h"
+
+namespace ton {
+
+Torrent::Torrent(TorrentMeta meta)
+    : info_(meta.info)
+    , merkle_tree_(info_.pieces_count(), info_.root_hash)
+    , piece_is_ready_(info_.pieces_count(), false) {
+  not_ready_piece_count_ = piece_is_ready_.size();
+  header_pieces_count_ = (info_.header_size + info_.piece_size - 1) / info_.piece_size;
+  not_ready_pending_piece_count_ = header_pieces_count_;
+
+  if (meta.header) {
+    set_header(meta.header.unwrap());
+  } else {
+    header_str_ = td::BufferSlice(info_.header_size);
+  }
+  if (meta.root_proof.not_null()) {
+    merkle_tree_.add_proof(meta.root_proof);
+  }
+}
+
+td::Result<Torrent> Torrent::open(Options options, TorrentMeta meta) {
+  Torrent res(std::move(meta));
+  if (!options.in_memory) {
+    if (options.root_dir.empty()) {
+      options.root_dir = ".";
+    }
+    res.set_root_dir(options.root_dir);
+  }
+  if (options.validate) {
+    res.validate();
+  }
+  return std::move(res);
+}
+
+td::Result<Torrent> Torrent::open(Options options, td::Slice meta_str) {
+  TRY_RESULT(meta, TorrentMeta::deserialize(meta_str));
+  return open(std::move(options), std::move(meta));
+}
+
+const Torrent::Info &Torrent::get_info() const {
+  return info_;
+}
+
+struct IterateInfo {
+  td::uint64 piece_offset;
+  td::uint64 chunk_offset;
+  td::uint64 size;
+};
+
+template <class F>
+td::Status Torrent::iterate_piece(Info::PieceInfo piece, F &&f) {
+  auto chunk_it = std::lower_bound(chunks_.begin(), chunks_.end(), piece.offset, [](auto &chunk, auto &piece_offset) {
+    return chunk.offset + chunk.size <= piece_offset;
+  });
+
+  td::uint64 size = 0;
+  for (; chunk_it != chunks_.end(); chunk_it++) {
+    if (chunk_it->offset >= piece.offset + piece.size) {
+      break;
+    }
+    if (chunk_it->size == 0) {
+      continue;
+    }
+    auto l = td::max(chunk_it->offset, piece.offset);
+    auto r = td::min(chunk_it->offset + chunk_it->size, piece.offset + piece.size);
+    CHECK(l < r);
+
+    IterateInfo info;
+    info.piece_offset = l - piece.offset;
+    info.chunk_offset = l - chunk_it->offset;
+    info.size = r - l;
+    size += info.size;
+    TRY_STATUS(f(chunk_it, info));
+  }
+  LOG_CHECK(size == piece.size) << size << " vs " << piece.size;
+  return td::Status::OK();
+}
+
+bool Torrent::is_piece_ready(td::uint64 piece_i) const {
+  CHECK(piece_i < info_.pieces_count());
+  return piece_is_ready_[piece_i];
+}
+
+td::optional<size_t> Torrent::get_files_count() const {
+  if (header_) {
+    return header_.value().files_count;
+  }
+  return {};
+}
+td::CSlice Torrent::get_file_name(size_t i) const {
+  return chunks_.at(i + 1).name;
+}
+td::uint64 Torrent::get_file_size(size_t i) const {
+  return chunks_.at(i + 1).size;
+}
+td::uint64 Torrent::get_file_ready_size(size_t i) const {
+  return chunks_.at(i + 1).ready_size;
+}
+
+Torrent::PartsRange Torrent::get_file_parts_range(size_t i) {
+  auto begin = chunks_.at(i + 1).offset;
+  auto end = begin + chunks_.at(i + 1).size;
+
+  PartsRange res;
+  res.begin = begin / info_.piece_size;
+  res.end = (end + info_.piece_size - 1) / info_.piece_size;
+  return res;
+}
+
+Torrent::PartsRange Torrent::get_header_parts_range() {
+  PartsRange res;
+  res.begin = 0;
+  res.end = header_pieces_count_;
+  return res;
+}
+
+TD_WARN_UNUSED_RESULT td::Status Torrent::ChunkState::get_piece(td::MutableSlice dest, td::uint64 offset,
+                                                                Cache *cache) {
+  if (dest.empty()) {
+    return td::Status::OK();
+  }
+
+  if (cache != nullptr) {
+    auto global_offset = offset + this->offset;
+    if (cache->offset > global_offset || cache->offset + cache->size < global_offset + dest.size()) {
+      auto load_size = td::min(size - offset, (td::uint64)cache->slice.size());
+      cache->size = 0;
+      TRY_STATUS(get_piece(cache->slice.as_slice().truncate(load_size), offset));
+      cache->offset = global_offset;
+      cache->size = load_size;
+    }
+
+    dest.copy_from(cache->slice.as_slice().substr(global_offset - cache->offset, dest.size()));
+    CHECK(cache->slice.size() >= dest.size());
+    return td::Status::OK();
+  }
+
+  TRY_RESULT(size, data.view_copy(dest, offset));
+  if (size != dest.size()) {
+    return td::Status::Error("Failed to read the whole chunk");
+  }
+  return td::Status::OK();
+}
+
+std::string Torrent::get_stats_str() const {
+  td::StringBuilder sb;
+  auto o_n = get_files_count();
+  if (!o_n) {
+    return "NO HEADER YET\n";
+  }
+  for (size_t i = 0, n = o_n.unwrap(); i < n; i++) {
+    auto size = get_file_size(i);
+    auto ready_size = get_file_ready_size(i);
+    sb << get_file_name(i) << "\t" << 100 * ready_size / size << "%%  " << td::format::as_size(ready_size) << "/"
+       << td::format::as_size(size) << "\n";
+  }
+  return sb.as_cslice().str();
+}
+
+void Torrent::validate() {
+  CHECK(header_);
+
+  std::fill(piece_is_ready_.begin(), piece_is_ready_.end(), false);
+  not_ready_piece_count_ = info_.pieces_count();
+
+  for (auto &chunk : chunks_) {
+    chunk.ready_size = 0;
+    if (root_dir_) {
+      if (td::stat(get_chunk_path(chunk.name)).is_error()) {
+        continue;
+      }
+    }
+    init_chunk_data(chunk);
+  }
+
+  std::vector<td::UInt256> hashes;
+  std::vector<MerkleTree::Chunk> chunks;
+
+  auto flush = [&] {
+    td::Bitset bitmask;
+    merkle_tree_.add_chunks(chunks, bitmask);
+    for (size_t i = 0; i < chunks.size(); i++) {
+      if (!bitmask.get(i)) {
+        continue;
+      }
+
+      auto piece_i = chunks[i].index;
+      auto piece = info_.get_piece_info(piece_i);
+      iterate_piece(piece, [&](auto it, auto info) {
+        it->ready_size += info.size;
+        return td::Status::OK();
+      });
+      piece_is_ready_[piece_i] = true;
+      ready_parts_count_++;
+
+      CHECK(not_ready_piece_count_);
+      not_ready_piece_count_--;
+    }
+
+    hashes.clear();
+    chunks.clear();
+  };
+
+  td::BufferSlice buf(info_.piece_size);
+  ChunkState::Cache cache;
+  cache.slice = td::BufferSlice(td::max(8u << 20, info_.piece_size));
+  for (size_t piece_i = 0; piece_i < info_.pieces_count(); piece_i++) {
+    auto piece = info_.get_piece_info(piece_i);
+    td::Sha256State sha256;
+    sha256.init();
+    bool skipped = false;
+    auto is_ok = iterate_piece(piece, [&](auto it, auto info) {
+      if (!it->data) {
+        skipped = true;
+        return td::Status::Error("No such file");
+      }
+      if (!it->has_piece(info.chunk_offset, info.size)) {
+        return td::Status::Error("Don't have piece");
+      }
+      auto dest = buf.as_slice().truncate(info.size);
+      TRY_STATUS(it->get_piece(dest, info.chunk_offset, &cache));
+      sha256.feed(dest);
+      //LOG(ERROR) << dest;
+      return td::Status::OK();
+    });
+    if (is_ok.is_error()) {
+      LOG_IF(ERROR, !skipped) << "Failed: " << is_ok;
+      LOG(ERROR) << "Failed: " << is_ok;
+      continue;
+    }
+    MerkleTree::Chunk chunk;
+    chunk.index = piece_i;
+    sha256.extract(chunk.hash.as_slice());
+
+    chunks.push_back(chunk);
+  }
+  flush();
+}
+
+td::Result<std::string> Torrent::get_piece_data(td::uint64 piece_i) {
+  CHECK(piece_i < info_.pieces_count());
+  if (!piece_is_ready_[piece_i]) {
+    return td::Status::Error("Piece is not ready");
+  }
+  auto it = pending_pieces_.find(piece_i);
+  if (it != pending_pieces_.end()) {
+    return it->second;
+  }
+  auto piece = info_.get_piece_info(piece_i);
+
+  std::string res(piece.size, '\0');
+  iterate_piece(piece, [&](auto it, auto info) {
+    return it->get_piece(td::MutableSlice(res).substr(info.piece_offset, info.size), info.chunk_offset);
+  });
+  return res;
+}
+
+td::Result<td::Ref<vm::Cell>> Torrent::get_piece_proof(td::uint64 piece_i) {
+  CHECK(piece_i < info_.pieces_count());
+  return merkle_tree_.gen_proof(piece_i, piece_i);
+}
+
+td::Status Torrent::add_piece(td::uint64 piece_i, td::Slice data, td::Ref<vm::Cell> proof) {
+  TRY_STATUS(merkle_tree_.add_proof(proof));
+  //LOG(ERROR) << "Add piece #" << piece_i;
+  CHECK(piece_i < info_.pieces_count());
+  if (piece_is_ready_[piece_i]) {
+    return td::Status::OK();
+  }
+  piece_is_ready_[piece_i] = true;
+  ready_parts_count_++;
+  ton::MerkleTree::Chunk chunk;
+  chunk.index = piece_i;
+  td::sha256(data, chunk.hash.as_slice());
+  TRY_STATUS(merkle_tree_.try_add_chunks({chunk}));
+
+  if (chunks_.empty()) {
+    return add_header_piece(piece_i, data);
+  }
+
+  return add_validated_piece(piece_i, data);
+}
+
+td::Status Torrent::add_header_piece(td::uint64 piece_i, td::Slice data) {
+  pending_pieces_[piece_i] = data.str();
+
+  if (piece_i < header_pieces_count_) {
+    //LOG(ERROR) << "Add header piece #" << piece_i;
+    auto piece = info_.get_piece_info(piece_i);
+    auto dest = header_str_.as_slice().substr(piece.offset);
+    data.truncate(dest.size());
+    dest.copy_from(data);
+    not_ready_pending_piece_count_--;
+    if (not_ready_pending_piece_count_ == 0) {
+      //LOG(ERROR) << "Got full header";
+      TorrentHeader header;
+      TRY_STATUS(td::unserialize(header, header_str_.as_slice()));  // TODO: it is a fatal error
+      set_header(header);
+      for (auto &it : pending_pieces_) {
+        TRY_STATUS(add_validated_piece(it.first, it.second));
+      }
+      pending_pieces_.clear();
+    }
+  } else {
+    LOG(ERROR) << "PENDING";
+  }
+
+  return td::Status::OK();
+}
+
+std::string Torrent::get_chunk_path(td::Slice name) {
+  return PSTRING() << root_dir_.value() << TD_DIR_SLASH << header_.value().dir_name << TD_DIR_SLASH << name;
+}
+
+td::Status Torrent::init_chunk_data(ChunkState &chunk) {
+  if (chunk.data) {
+    return td::Status::OK();
+  }
+  if (root_dir_) {
+    TRY_RESULT(data, td::FileNoCacheBlobView::create(get_chunk_path(chunk.name), chunk.size, true));
+    chunk.data = std::move(data);
+  } else {
+    chunk.data = td::BufferSliceBlobView::create(td::BufferSlice(chunk.size));
+  }
+  return td::Status::OK();
+}
+
+td::Status Torrent::add_validated_piece(td::uint64 piece_i, td::Slice data) {
+  CHECK(!chunks_.empty());
+
+  auto piece = info_.get_piece_info(piece_i);
+
+  TRY_STATUS(iterate_piece(piece, [&](auto it, auto info) {
+    TRY_STATUS(init_chunk_data(*it));
+    return it->add_piece(data.substr(info.piece_offset, info.size), info.chunk_offset);
+  }));
+  piece_is_ready_[piece_i] = true;
+  not_ready_piece_count_--;
+
+  return td::Status::OK();
+}
+
+bool Torrent::is_completed() const {
+  return not_ready_piece_count_ == 0;
+}
+
+td::Result<td::BufferSlice> Torrent::read_file(td::Slice name) {
+  for (auto &chunk : chunks_) {
+    if (chunk.name == name) {
+      td::BufferSlice res(chunk.size);
+      TRY_STATUS(chunk.get_piece(res.as_slice(), 0));
+      return std::move(res);
+    }
+  }
+  return td::Status::Error("Unknown name");
+}
+
+Torrent::GetMetaOptions::GetMetaOptions() = default;
+std::string Torrent::get_meta_str(const GetMetaOptions &options) const {
+  return get_meta(options).serialize();
+}
+
+TorrentMeta Torrent::get_meta(const GetMetaOptions &options) const {
+  TorrentMeta torrent_file;
+  if (options.with_header) {
+    torrent_file.header = header_;
+  }
+  torrent_file.info = info_;
+  torrent_file.info.init_cell();
+  if (options.with_proof) {
+    torrent_file.root_proof = merkle_tree_.get_root(options.proof_depth_limit);
+  }
+  return torrent_file;
+}
+
+Torrent::Torrent(Info info, td::optional<TorrentHeader> header, ton::MerkleTree tree, std::vector<ChunkState> chunks)
+    : info_(info)
+    , header_(std::move(header))
+    , merkle_tree_(std::move(tree))
+    , piece_is_ready_(info_.pieces_count(), true)
+    , ready_parts_count_{info_.pieces_count()}
+    , chunks_(std::move(chunks)) {
+}
+
+void Torrent::set_header(const TorrentHeader &header) {
+  header_ = header;
+  auto add_chunk = [&](td::Slice name, td::uint64 offset, td::uint64 size) {
+    ChunkState chunk;
+    chunk.name = name.str();
+    chunk.ready_size = 0;
+    chunk.size = size;
+    chunk.offset = offset;
+    chunks_.push_back(std::move(chunk));
+  };
+  add_chunk("", 0, header.serialization_size());
+  chunks_.back().data = td::BufferSliceBlobView::create(header.serialize());
+  for (size_t i = 0; i < header.files_count; i++) {
+    auto l = header.get_data_begin(i);
+    auto r = header.get_data_end(i);
+    add_chunk(header.get_name(i), l, r - l);
+  }
+}
+
+size_t Torrent::get_ready_parts_count() const {
+  return ready_parts_count_;
+}
+
+std::vector<size_t> Torrent::chunks_by_piece(td::uint64 piece_id) {
+  std::vector<size_t> res;
+  auto piece = info_.get_piece_info(piece_id);
+  auto is_ok = iterate_piece(piece, [&](auto it, auto info) {
+    res.push_back(it - chunks_.begin());
+    return td::Status::OK();
+  });
+  return res;
+}
+
+}  // namespace ton

storage/Torrent.h

@@ -0,0 +1,150 @@
+#pragma once
+#include "MerkleTree.h"
+#include "TorrentMeta.h"
+
+#include "td/utils/buffer.h"
+#include "td/db/utils/BlobView.h"
+
+#include <map>
+
+namespace ton {
+class Torrent {
+ public:
+  class Creator;
+  friend class Creator;
+  using Info = TorrentInfo;
+
+  struct Options {
+    std::string root_dir;
+    bool in_memory{false};
+    bool validate{false};
+  };
+
+  // creation
+  static td::Result<Torrent> open(Options options, TorrentMeta meta);
+  static td::Result<Torrent> open(Options options, td::Slice meta_str);
+  void validate();
+
+  std::string get_stats_str() const;
+
+  const Info &get_info() const;
+
+  // get piece and proof
+  td::Result<std::string> get_piece_data(td::uint64 piece_i);
+  td::Result<td::Ref<vm::Cell>> get_piece_proof(td::uint64 piece_i);
+
+  // add piece (with an optional proof)
+  td::Status add_piece(td::uint64 piece_i, td::Slice data, td::Ref<vm::Cell> proof);
+  //TODO: add multiple chunks? Merkle tree supports much more general interface
+
+  bool is_completed() const;
+
+  // Checks that file is ready and returns its content.
+  // Intened mostly for in-memory usage and for tests
+  td::Result<td::BufferSlice> read_file(td::Slice name);
+
+  struct GetMetaOptions {
+    GetMetaOptions();
+    size_t proof_depth_limit{std::numeric_limits<size_t>::max()};
+    bool with_header{true};
+    bool with_proof{true};
+
+    GetMetaOptions &without_header() {
+      with_header = false;
+      return *this;
+    }
+    GetMetaOptions &without_proof() {
+      with_proof = false;
+      return *this;
+    }
+    GetMetaOptions &with_proof_depth_limit(size_t limit) {
+      proof_depth_limit = limit;
+      return *this;
+    }
+  };
+  std::string get_meta_str(const GetMetaOptions &options = {}) const;
+  TorrentMeta get_meta(const GetMetaOptions &options = {}) const;
+
+  // Some api for inspection of a current state
+  bool is_piece_ready(td::uint64 piece_i) const;
+
+  td::optional<size_t> get_files_count() const;
+  td::CSlice get_file_name(size_t i) const;
+  td::uint64 get_file_size(size_t i) const;
+  td::uint64 get_file_ready_size(size_t i) const;
+
+  struct PartsRange {
+    td::uint64 begin{0};
+    td::uint64 end{0};
+  };
+  PartsRange get_file_parts_range(size_t i);
+  PartsRange get_header_parts_range();
+
+  size_t get_ready_parts_count() const;
+
+  std::vector<size_t> chunks_by_piece(td::uint64 piece_id);
+
+ private:
+  Info info_;
+  td::optional<std::string> root_dir_;
+
+  // While header is not completly available all pieces are stored in memory
+  td::BufferSlice header_str_;
+  td::optional<TorrentHeader> header_;
+  size_t not_ready_pending_piece_count_{0};
+  size_t header_pieces_count_{0};
+  std::map<td::uint64, td::string> pending_pieces_;
+
+  ton::MerkleTree merkle_tree_;
+
+  std::vector<bool> piece_is_ready_;
+  size_t not_ready_piece_count_{0};
+  size_t ready_parts_count_{0};
+
+  struct ChunkState {
+    std::string name;
+    td::uint64 offset{0};
+    td::uint64 size{0};
+    td::uint64 ready_size{0};
+    td::BlobView data;
+
+    struct Cache {
+      td::uint64 offset{0};
+      td::uint64 size{0};
+      td::BufferSlice slice;
+    };
+
+    bool is_ready() const {
+      return ready_size == size;
+    }
+
+    TD_WARN_UNUSED_RESULT td::Status add_piece(td::Slice piece, td::uint64 offset) {
+      TRY_RESULT(written, data.write(piece, offset));
+      CHECK(written == piece.size());
+      ready_size += written;
+      return td::Status::OK();
+    }
+    bool has_piece(td::uint64 offset, td::uint64 size) {
+      return data.size() >= offset + size;
+    }
+    TD_WARN_UNUSED_RESULT td::Status get_piece(td::MutableSlice dest, td::uint64 offset, Cache *cache = nullptr);
+  };
+  std::vector<ChunkState> chunks_;
+
+  explicit Torrent(Info info, td::optional<TorrentHeader> header, ton::MerkleTree tree, std::vector<ChunkState> chunk);
+  explicit Torrent(TorrentMeta meta);
+  void set_root_dir(std::string root_dir) {
+    root_dir_ = std::move(root_dir);
+  }
+
+  std::string get_chunk_path(td::Slice name);
+  td::Status init_chunk_data(ChunkState &chunk);
+  template <class F>
+  td::Status iterate_piece(Info::PieceInfo piece, F &&f);
+
+  td::Status add_header_piece(td::uint64 piece_i, td::Slice data);
+  td::Status add_validated_piece(td::uint64 piece_i, td::Slice data);
+  void set_header(const TorrentHeader &header);
+};
+
+}  // namespace ton

storage/TorrentCreator.cpp

@@ -0,0 +1,173 @@
+#include "TorrentCreator.h"
+
+#include "td/db/utils/CyclicBuffer.h"
+
+#include "td/utils/crypto.h"
+#include "td/utils/PathView.h"
+#include "td/utils/port/path.h"
+#include "td/utils/tl_helpers.h"
+
+namespace ton {
+td::Result<Torrent> Torrent::Creator::create_from_path(Options options, td::CSlice raw_path) {
+  TRY_RESULT(path, td::realpath(raw_path));
+  TRY_RESULT(stat, td::stat(path));
+  if (stat.is_dir_) {
+    if (!path.empty() && path.back() != TD_DIR_SLASH) {
+      path += TD_DIR_SLASH;
+    }
+    if (!options.dir_name) {
+      options.dir_name = td::PathView::dir_and_file(path).str();
+    }
+    Torrent::Creator creator(options);
+    td::Status status;
+    auto walk_status = td::WalkPath::run(path, [&](td::CSlice name, td::WalkPath::Type type) {
+      if (type == td::WalkPath::Type::NotDir) {
+        status = creator.add_file(td::PathView::relative(name, path), name);
+        if (status.is_error()) {
+          return td::WalkPath::Action::Abort;
+        }
+      }
+      return td::WalkPath::Action::Continue;
+    });
+    TRY_STATUS(std::move(status));
+    TRY_STATUS(std::move(walk_status));
+    return creator.finalize();
+  } else {
+    Torrent::Creator creator(options);
+    TRY_STATUS(creator.add_file(td::PathView(path).file_name(), path));
+    return creator.finalize();
+  }
+}
+
+td::Result<Torrent> Torrent::Creator::create_from_blobs(Options options, td::Span<Blob> blobs) {
+  Torrent::Creator creator(options);
+  for (auto &blob : blobs) {
+    TRY_STATUS(creator.add_blob(blob.name, blob.data));
+  }
+  return creator.finalize();
+}
+
+td::Status Torrent::Creator::add_blob(td::Slice name, td::Slice blob) {
+  return add_blob(name, td::BufferSliceBlobView::create(td::BufferSlice(blob)));
+}
+
+td::Status Torrent::Creator::add_blob(td::Slice name, td::BlobView blob) {
+  File file;
+  file.name = name.str();
+  file.data = std::move(blob);
+  files_.push_back(std::move(file));
+  return td::Status::OK();
+}
+
+TD_WARN_UNUSED_RESULT td::Status Torrent::Creator::add_file(td::Slice name, td::CSlice path) {
+  LOG(INFO) << "Add file " << name << " " << path;
+  TRY_RESULT(data, td::FileNoCacheBlobView::create(path));
+  return add_blob(name, std::move(data));
+}
+
+td::Result<Torrent> Torrent::Creator::finalize() {
+  TorrentHeader header;
+  TRY_RESULT(files_count, td::narrow_cast_safe<td::uint32>(files_.size()));
+  header.files_count = files_count;
+  header.data_index.resize(files_count);
+  header.name_index.resize(files_count);
+  td::uint64 data_offset = 0;
+  for (size_t i = 0; i < files_.size(); i++) {
+    header.names += files_[i].name;
+    header.name_index[i] = header.names.size();
+    data_offset += files_[i].data.size();
+    header.data_index[i] = data_offset;
+  }
+  header.tot_names_size = header.names.size();
+  if (options_.dir_name) {
+    header.dir_name = options_.dir_name.value();
+  }
+
+  // Now we should stream all data to calculate sha256 of all pieces
+
+  std::string buffer;
+  td::CyclicBuffer::Options cb_options;
+  cb_options.chunk_size = td::max(options_.piece_size * 16, (1 << 20) / options_.piece_size * options_.piece_size);
+  cb_options.count = 2;
+
+  auto reader_writer = td::CyclicBuffer::create(cb_options);
+  auto reader = std::move(reader_writer.first);
+  auto writer = std::move(reader_writer.second);
+
+  auto header_size = header.serialization_size();
+  auto file_size = header_size + data_offset;
+  auto chunks_count = (file_size + options_.piece_size - 1) / options_.piece_size;
+  ton::MerkleTree tree;
+  tree.init_begin(chunks_count);
+  std::vector<Torrent::ChunkState> chunks;
+  size_t chunk_i = 0;
+  auto flush_reader = [&](bool force) {
+    while (true) {
+      auto slice = reader.prepare_read();
+      slice.truncate(options_.piece_size);
+      if (slice.empty() || (slice.size() != options_.piece_size && !force)) {
+        break;
+      }
+      td::UInt256 hash;
+      sha256(slice, hash.as_slice());
+      CHECK(chunk_i < chunks_count);
+      tree.init_add_chunk(chunk_i, hash.as_slice());
+      chunk_i++;
+      reader.confirm_read(slice.size());
+    }
+  };
+  td::uint64 offset = 0;
+  auto add_blob = [&](auto &&data, td::Slice name) {
+    td::uint64 data_offset = 0;
+    while (data_offset < data.size()) {
+      auto dest = writer.prepare_write();
+      CHECK(dest.size() != 0);
+      dest.truncate(data.size() - data_offset);
+      TRY_RESULT(got_size, data.view_copy(dest, data_offset));
+      CHECK(got_size != 0);
+      data_offset += got_size;
+      writer.confirm_write(got_size);
+      flush_reader(false);
+    }
+
+    Torrent::ChunkState chunk;
+    chunk.name = name.str();
+    chunk.offset = offset;
+    chunk.size = data.size();
+    chunk.ready_size = chunk.size;
+    chunk.data = std::move(data);
+
+    offset += chunk.size;
+    chunks.push_back(std::move(chunk));
+    return td::Status::OK();
+  };
+
+  Torrent::Info info;
+  auto header_str = td::serialize(header);
+  CHECK(header_size == header_str.size());
+  info.header_size = header_str.size();
+  td::sha256(header_str, info.header_hash.as_slice());
+
+  add_blob(td::BufferSliceBlobView::create(td::BufferSlice(header_str)), "").ensure();
+  for (auto &file : files_) {
+    add_blob(std::move(file.data), file.name).ensure();
+  }
+  flush_reader(true);
+  tree.init_finish();
+  CHECK(chunk_i == chunks_count);
+  CHECK(offset == file_size);
+
+  info.header_size = header.serialization_size();
+  info.piece_size = options_.piece_size;
+  info.description = options_.description;
+  info.file_size = file_size;
+  info.depth = tree.get_depth();
+  info.root_hash = tree.get_root_hash();
+
+  info.init_cell();
+
+  Torrent torrent(info, std::move(header), std::move(tree), std::move(chunks));
+
+  return std::move(torrent);
+}
+}  // namespace ton

storage/TorrentCreator.h

@@ -0,0 +1,48 @@
+#pragma once
+#include "Torrent.h"
+
+#include "td/utils/optional.h"
+#include "td/utils/SharedSlice.h"
+#include "td/db/utils/BlobView.h"
+
+namespace ton {
+class Torrent::Creator {
+ public:
+  struct Options {
+    td::uint32 piece_size{128 * 768};
+
+    // override default dir_name
+    // should't be used in a usual workflow
+    td::optional<std::string> dir_name;
+
+    std::string description;
+  };
+
+  // If path is a file create a torrent with one file in it.
+  // If path is a directory, create a torrent with whole directory.
+  static td::Result<Torrent> create_from_path(Options options, td::CSlice path);
+  struct Blob {
+    td::Slice name;
+    td::Slice data;
+  };
+  static td::Result<Torrent> create_from_blobs(Options options, td::Span<Blob> blobs);
+
+  // This api is mostly for internal usage. One should prefer static methods
+  explicit Creator(Options options) : options_(options) {
+  }
+  TD_WARN_UNUSED_RESULT td::Status add_blob(td::Slice name, td::Slice blob);
+  TD_WARN_UNUSED_RESULT td::Status add_blob(td::Slice name, td::BlobView blob);
+  TD_WARN_UNUSED_RESULT td::Status add_file(td::Slice name, td::CSlice path);
+  td::Result<Torrent> finalize();
+
+ private:
+  td::Status init();
+
+  Options options_;
+  struct File {
+    std::string name;
+    td::BlobView data;
+  };
+  std::vector<File> files_;
+};
+}  // namespace ton

storage/TorrentHeader.cpp

@@ -0,0 +1,50 @@
+#include "TorrentHeader.hpp"
+
+#include "td/utils/tl_helpers.h"
+
+namespace ton {
+td::CSlice TorrentHeader::get_dir_name() const {
+  return dir_name;
+}
+
+td::uint32 TorrentHeader::get_files_count() const {
+  return files_count;
+}
+
+td::uint64 TorrentHeader::get_data_begin(td::uint64 file_i) const {
+  return get_data_offset(file_i);
+}
+td::uint64 TorrentHeader::get_data_end(td::uint64 file_i) const {
+  return get_data_offset(file_i + 1);
+}
+
+td::uint64 TorrentHeader::serialization_size() const {
+  return td::tl_calc_length(*this);
+}
+td::uint64 TorrentHeader::get_data_offset(td::uint64 offset_i) const {
+  td::uint64 res = serialization_size();
+  if (offset_i > 0) {
+    CHECK(offset_i <= files_count);
+    res += data_index[offset_i - 1];
+  }
+  return res;
+}
+td::BufferSlice TorrentHeader::serialize() const {
+  return td::BufferSlice(td::serialize(*this));
+}
+td::uint64 TorrentHeader::get_data_size(td::uint64 file_i) const {
+  auto res = data_index[file_i];
+  if (file_i > 0) {
+    res -= data_index[file_i - 1];
+  }
+  return res;
+}
+
+td::Slice TorrentHeader::get_name(td::uint64 file_i) const {
+  CHECK(file_i < files_count);
+  auto from = file_i == 0 ? 0 : name_index[file_i - 1];
+  auto till = name_index[file_i];
+  return td::Slice(names).substr(from, till - from);
+}
+
+}  // namespace ton

storage/TorrentHeader.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include "td/utils/Slice.h"
+#include "td/utils/buffer.h"
+
+namespace ton {
+// fec_info_none#c82a1964 = FecInfo;
+//
+// torrent_header#9128aab7
+//   files_count:uint32
+//   tot_name_size:uint64
+//   tot_data_size:uint64
+//   fec:FecInfo
+//   dir_name_size:uint32
+//   dir_name:(dir_name_size * [uint8])
+//   name_index:(files_count * [uint64])
+//   data_index:(files_count * [uint64])
+//   names:(file_names_size * [uint8])
+//   data:(tot_data_size * [uint8])
+//     = TorrentHeader;
+
+struct TorrentHeader {
+  td::uint32 files_count{0};
+  td::uint64 tot_names_size{0};
+  td::uint64 tot_data_size{0};
+  //fec_none
+  std::string dir_name;
+  std::vector<td::uint64> name_index;
+  std::vector<td::uint64> data_index;
+  std::string names;
+
+  td::uint64 get_data_begin(td::uint64 file_i) const;
+  td::uint64 get_data_end(td::uint64 file_i) const;
+  td::uint64 get_data_offset(td::uint64 offset_i) const;
+  td::uint64 get_data_size(td::uint64 file_i) const;
+  td::Slice get_name(td::uint64 file_i) const;
+  td::CSlice get_dir_name() const;
+  td::uint32 get_files_count() const;
+
+  td::uint64 serialization_size() const;
+  td::BufferSlice serialize() const;
+
+  static constexpr td::uint32 type = 0x9128aab7;
+  template <class StorerT>
+  void store(StorerT &storer) const;
+  template <class ParserT>
+  void parse(ParserT &parser);
+};
+}  // namespace ton