Merge branch 'testnet' into block-generation

adnl/adnl-peer.cpp

@@ -960,7 +960,6 @@ void AdnlPeerPairImpl::got_data_from_dht(td::Result<AdnlNode> R) {
   CHECK(dht_query_active_);
   dht_query_active_ = false;
   next_dht_query_at_ = td::Timestamp::in(td::Random::fast(60.0, 120.0));
-  alarm_timestamp().relax(next_dht_query_at_);
   if (R.is_error()) {
     VLOG(ADNL_INFO) << this << ": dht query failed: " << R.move_as_error();
     return;

crypto/block/transaction.cpp

@@ -704,7 +704,9 @@ bool Transaction::prepare_storage_phase(const StoragePhaseConfig& cfg, bool forc
       switch (acc_status) {
         case Account::acc_uninit:
         case Account::acc_frozen:
-          if (total_due > cfg.delete_due_limit) {
+          if (total_due > cfg.delete_due_limit && balance.extra.is_null()) {
+            // Keeping accounts with non-null extras is a temporary measure before implementing proper collection of
+            // extracurrencies from deleted accounts
             res->deleted = true;
             acc_status = Account::acc_deleted;
             if (balance.extra.not_null()) {

crypto/vm/db/CellHashTable.h

@@ -43,7 +43,7 @@ class CellHashTable {
   template <class F>
   void for_each(F &&f) {
     for (auto &info : set_) {
-      f(const_cast<InfoT &>(info));
+      f(info);
     }
   }
   template <class F>

crypto/vm/db/CellStorage.cpp

@@ -150,20 +150,6 @@ td::Result<CellLoader::LoadResult> CellLoader::load(td::Slice hash, bool need_da
   return res;
 }
 
-td::Result<CellLoader::LoadResult> CellLoader::load_refcnt(td::Slice hash) {
-  LoadResult res;
-  std::string serialized;
-  TRY_RESULT(get_status, reader_->get(hash, serialized));
-  if (get_status != KeyValue::GetStatus::Ok) {
-    DCHECK(get_status == KeyValue::GetStatus::NotFound);
-    return res;
-  }
-  res.status = LoadResult::Ok;
-  td::TlParser parser(serialized);
-  td::parse(res.refcnt_, parser);
-  return res;
-}
-
 CellStorer::CellStorer(KeyValue &kv) : kv_(kv) {
 }
 

crypto/vm/db/CellStorage.h

@@ -48,7 +48,6 @@ class CellLoader {
   };
   CellLoader(std::shared_ptr<KeyValueReader> reader);
   td::Result<LoadResult> load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator);
-  td::Result<LoadResult> load_refcnt(td::Slice hash);  // This only loads refcnt_, cell_ == null
 
  private:
   std::shared_ptr<KeyValueReader> reader_;

crypto/vm/db/DynamicBagOfCellsDb.cpp

@@ -27,9 +27,6 @@
 #include "td/utils/ThreadSafeCounter.h"
 
 #include "vm/cellslice.h"
-#include <queue>
-#include "td/actor/actor.h"
-#include "common/delay.h"
 
 namespace vm {
 namespace {
@@ -558,177 +555,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
                                               DynamicBocExtCellExtra{cell_db_reader_}));
     return std::move(res);
   }
-
-  struct PrepareCommitAsyncState {
-    size_t remaining_ = 0;
-    std::shared_ptr<AsyncExecutor> executor_;
-    td::Promise<td::Unit> promise_;
-
-    struct CellInfo2 {
-      CellInfo *info;
-      std::vector<CellInfo2 *> parents;
-      unsigned remaining_children = 0;
-      Cell::Hash key() const {
-        return info->key();
-      }
-      bool operator<(const CellInfo2 &other) const {
-        return key() < other.key();
-      }
-
-      friend bool operator<(const CellInfo2 &a, td::Slice b) {
-        return a.key().as_slice() < b;
-      }
-
-      friend bool operator<(td::Slice a, const CellInfo2 &b) {
-        return a < b.key().as_slice();
-      }
-    };
-
-    CellHashTable<CellInfo2> cells_;
-  };
-  std::unique_ptr<PrepareCommitAsyncState> pca_state_;
-
-  void prepare_commit_async(std::shared_ptr<AsyncExecutor> executor, td::Promise<td::Unit> promise) override {
-    if (pca_state_) {
-      promise.set_error(td::Status::Error("Other prepare_commit_async is not finished"));
-      return;
-    }
-    if (is_prepared_for_commit()) {
-      promise.set_result(td::Unit());
-      return;
-    }
-    pca_state_ = std::make_unique<PrepareCommitAsyncState>();
-    pca_state_->executor_ = std::move(executor);
-    pca_state_->promise_ = std::move(promise);
-    for (auto &new_cell : to_inc_) {
-      dfs_new_cells_in_db_async(new_cell);
-    }
-    pca_state_->cells_.for_each([&](PrepareCommitAsyncState::CellInfo2 &info) {
-      ++pca_state_->remaining_;
-      if (info.remaining_children == 0) {
-        pca_load_from_db(&info);
-      }
-    });
-    if (pca_state_->remaining_ == 0) {
-      prepare_commit_async_cont();
-    }
-  }
-
-  void dfs_new_cells_in_db_async(const td::Ref<vm::Cell> &cell, PrepareCommitAsyncState::CellInfo2 *parent = nullptr) {
-    bool exists = true;
-    pca_state_->cells_.apply(cell->get_hash().as_slice(), [&](PrepareCommitAsyncState::CellInfo2 &info) {
-      if (info.info == nullptr) {
-        exists = false;
-        info.info = &get_cell_info(cell);
-      }
-    });
-    auto info = pca_state_->cells_.get_if_exists(cell->get_hash().as_slice());
-    if (parent) {
-      info->parents.push_back(parent);
-      ++parent->remaining_children;
-    }
-    if (exists) {
-      return;
-    }
-    if (cell->is_loaded()) {
-      vm::CellSlice cs(vm::NoVm{}, cell);
-      for (unsigned i = 0; i < cs.size_refs(); i++) {
-        dfs_new_cells_in_db_async(cs.prefetch_ref(i), info);
-      }
-    }
-  }
-
-  void pca_load_from_db(PrepareCommitAsyncState::CellInfo2 *info) {
-    pca_state_->executor_->execute_async(
-        [db = this, info, executor = pca_state_->executor_, loader = *loader_]() mutable {
-          auto res = loader.load_refcnt(info->info->cell->get_hash().as_slice()).move_as_ok();
-          executor->execute_sync([db, info, res = std::move(res)]() {
-            db->pca_set_in_db(info, std::move(res));
-          });
-        });
-  }
-
-  void pca_set_in_db(PrepareCommitAsyncState::CellInfo2 *info, CellLoader::LoadResult result) {
-    info->info->sync_with_db = true;
-    if (result.status == CellLoader::LoadResult::Ok) {
-      info->info->in_db = true;
-      info->info->db_refcnt = result.refcnt();
-    } else {
-      info->info->in_db = false;
-    }
-    for (PrepareCommitAsyncState::CellInfo2 *parent_info : info->parents) {
-      if (parent_info->info->sync_with_db) {
-        continue;
-      }
-      if (!info->info->in_db) {
-        pca_set_in_db(parent_info, {});
-      } else if (--parent_info->remaining_children == 0) {
-        pca_load_from_db(parent_info);
-      }
-    }
-    if (--pca_state_->remaining_ == 0) {
-      prepare_commit_async_cont();
-    }
-  }
-
-  void prepare_commit_async_cont() {
-    for (auto &new_cell : to_inc_) {
-      auto &new_cell_info = get_cell_info(new_cell);
-      dfs_new_cells(new_cell_info);
-    }
-
-    CHECK(pca_state_->remaining_ == 0);
-    for (auto &old_cell : to_dec_) {
-      auto &old_cell_info = get_cell_info(old_cell);
-      dfs_old_cells_async(old_cell_info);
-    }
-    if (pca_state_->remaining_ == 0) {
-      prepare_commit_async_cont2();
-    }
-  }
-
-  void dfs_old_cells_async(CellInfo &info) {
-    if (!info.was) {
-      info.was = true;
-      visited_.push_back(&info);
-      if (!info.sync_with_db) {
-        ++pca_state_->remaining_;
-        load_cell_async(
-            info.cell->get_hash().as_slice(), pca_state_->executor_,
-            [executor = pca_state_->executor_, db = this, info = &info](td::Result<td::Ref<vm::DataCell>> R) {
-              R.ensure();
-              executor->execute_sync([db, info]() {
-                CHECK(info->sync_with_db);
-                db->dfs_old_cells_async(*info);
-                if (--db->pca_state_->remaining_ == 0) {
-                  db->prepare_commit_async_cont2();
-                }
-              });
-            });
-        return;
-      }
-    }
-    info.refcnt_diff--;
-    if (!info.sync_with_db) {
-      return;
-    }
-    auto new_refcnt = info.refcnt_diff + info.db_refcnt;
-    CHECK(new_refcnt >= 0);
-    if (new_refcnt != 0) {
-      return;
-    }
-
-    for_each(info, [this](auto &child_info) { dfs_old_cells_async(child_info); });
-  }
-
-  void prepare_commit_async_cont2() {
-    save_diff_prepare();
-    to_inc_.clear();
-    to_dec_.clear();
-    pca_state_->promise_.set_result(td::Unit());
-    pca_state_ = {};
-  }
-
 };
 }  // namespace
 

crypto/vm/db/DynamicBagOfCellsDb.h

@@ -75,7 +75,6 @@ class DynamicBagOfCellsDb {
 
   virtual void load_cell_async(td::Slice hash, std::shared_ptr<AsyncExecutor> executor,
                                td::Promise<Ref<DataCell>> promise) = 0;
-  virtual void prepare_commit_async(std::shared_ptr<AsyncExecutor> executor, td::Promise<td::Unit> promise) = 0;
 };
 
 }  // namespace vm

dht/dht-remote-node.cpp

@@ -140,13 +140,7 @@ adnl::AdnlNodeIdFull DhtRemoteNode::get_full_id() const {
 
 td::Result<std::unique_ptr<DhtRemoteNode>> DhtRemoteNode::create(DhtNode node, td::uint32 max_missed_pings,
                                                                  td::int32 our_network_id) {
-  TRY_RESULT(enc, node.adnl_id().pubkey().create_encryptor());
+  TRY_STATUS(node.check_signature());
-  auto tl = node.tl();
-  auto sig = std::move(tl->signature_);
-
-  TRY_STATUS_PREFIX(enc->check_signature(serialize_tl_object(tl, true).as_slice(), sig.as_slice()),
-                    "bad node signature: ");
-
   return std::make_unique<DhtRemoteNode>(std::move(node), max_missed_pings, our_network_id);
 }
 

utils/generate-random-id.cpp

@@ -28,9 +28,6 @@
 #include <iostream>
 #include <iomanip>
 #include <string>
-#include <cstring>
-#include <cassert>
-#include "crypto/ellcurve/Ed25519.h"
 #include "adnl/utils.hpp"
 #include "auto/tl/ton_api.h"
 #include "auto/tl/ton_api_json.h"
@@ -38,12 +35,13 @@
 #include "td/utils/OptionParser.h"
 #include "td/utils/filesystem.h"
 #include "keys/encryptor.h"
-#include "keys/keys.hpp"
 #include "git.h"
+#include "dht/dht-node.hpp"
 
 int main(int argc, char *argv[]) {
   ton::PrivateKey pk;
-  ton::tl_object_ptr<ton::ton_api::adnl_addressList> addr_list;
+  td::optional<ton::adnl::AdnlAddressList> addr_list;
+  td::optional<td::int32> network_id_opt;
 
   td::OptionParser p;
   p.set_description("generate random id");
@@ -78,11 +76,19 @@ int main(int argc, char *argv[]) {
     if (addr_list) {
       return td::Status::Error("duplicate '-a' option");
     }
-    CHECK(!addr_list);
 
     td::BufferSlice bs(key);
     TRY_RESULT_PREFIX(as_json_value, td::json_decode(bs.as_slice()), "bad addr list JSON: ");
-    TRY_STATUS_PREFIX(td::from_json(addr_list, std::move(as_json_value)), "bad addr list TL: ");
+    ton::tl_object_ptr<ton::ton_api::adnl_addressList> addr_list_tl;
+    TRY_STATUS_PREFIX(td::from_json(addr_list_tl, std::move(as_json_value)), "bad addr list TL: ");
+    TRY_RESULT_PREFIX_ASSIGN(addr_list, ton::adnl::AdnlAddressList::create(addr_list_tl), "bad addr list: ");
+    return td::Status::OK();
+  });
+  p.add_checked_option('i', "network-id", "dht network id (default: -1)", [&](td::Slice key) {
+    if (network_id_opt) {
+      return td::Status::Error("duplicate '-i' option");
+    }
+    TRY_RESULT_PREFIX_ASSIGN(network_id_opt, td::to_integer_safe<td::int32>(key), "bad network id: ");
     return td::Status::OK();
   });
 
@@ -118,7 +124,7 @@ int main(int argc, char *argv[]) {
       std::cerr << "'-a' option missing" << std::endl;
       return 2;
     }
-    auto x = ton::create_tl_object<ton::ton_api::adnl_node>(pub_key.tl(), std::move(addr_list));
+    auto x = ton::create_tl_object<ton::ton_api::adnl_node>(pub_key.tl(), addr_list.value().tl());
     auto e = pk.create_decryptor().move_as_ok();
     auto r = e->sign(ton::serialize_tl_object(x, true).as_slice()).move_as_ok();
 
@@ -129,12 +135,17 @@ int main(int argc, char *argv[]) {
       std::cerr << "'-a' option missing" << std::endl;
       return 2;
     }
-    auto x = ton::create_tl_object<ton::ton_api::dht_node>(pub_key.tl(), std::move(addr_list), -1, td::BufferSlice());
+    td::int32 network_id = network_id_opt ? network_id_opt.value() : -1;
+    td::BufferSlice to_sign = ton::serialize_tl_object(
+        ton::dht::DhtNode{ton::adnl::AdnlNodeIdFull{pub_key}, addr_list.value(), -1, network_id, td::BufferSlice{}}
+            .tl(),
+        true);
     auto e = pk.create_decryptor().move_as_ok();
-    auto r = e->sign(ton::serialize_tl_object(x, true).as_slice()).move_as_ok();
+    auto signature = e->sign(to_sign.as_slice()).move_as_ok();
-    x->signature_ = std::move(r);
+    auto node =
+        ton::dht::DhtNode{ton::adnl::AdnlNodeIdFull{pub_key}, addr_list.value(), -1, network_id, std::move(signature)};
 
-    auto v = td::json_encode<std::string>(td::ToJson(x));
+    auto v = td::json_encode<std::string>(td::ToJson(node.tl()));
     std::cout << v << "\n";
   } else if (mode == "keys") {
     td::write_file(name, pk.export_as_slice()).ensure();

validator/db/celldb.cpp

@@ -86,42 +86,19 @@ void CellDbIn::start_up() {
 }
 
 void CellDbIn::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
-  enqueue([this, hash, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+  boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise));
-    if (R.is_error()) {
-      return;
-    }
-    promise.set_result(boc_->load_cell(hash.as_slice()));
-    release_db();
-  });
 }
 
 void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise) {
-  enqueue([this, block_id, cell = std::move(cell), promise = std::move(promise)](td::Result<td::Unit> R0) mutable {
+  td::PerfWarningTimer timer{"storecell", 0.1};
-    if (R0.is_error()) {
-      return;
-    }
-    promise = promise.wrap([timer = td::PerfWarningTimer{"storecell", 0.1}](td::Ref<vm::DataCell> &&r) { return r; });
-    auto key_hash = get_key_hash(block_id);
-    auto R = get_block(key_hash);
-    // duplicate
-    if (R.is_ok()) {
-      promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
-      release_db();
-      return;
-    }
-
-    boc_->inc(cell);
-    boc_->prepare_commit_async(
-        async_executor, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::Unit> R) mutable {
-          R.ensure();
-          td::actor::send_closure(SelfId, &CellDbIn::store_cell_cont, block_id, cell, std::move(promise));
-        });
-  });
-}
-
-void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell,
-                               td::Promise<td::Ref<vm::DataCell>> promise) {
   auto key_hash = get_key_hash(block_id);
+  auto R = get_block(key_hash);
+  // duplicate
+  if (R.is_ok()) {
+    promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
+    return;
+  }
+
   auto empty = get_empty_key_hash();
   auto ER = get_block(empty);
   ER.ensure();
@@ -142,7 +119,9 @@ void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell,
     P.prev = key_hash;
   }
 
-  vm::CellStorer stor{*cell_db_};
+  boc_->inc(cell);
+  boc_->prepare_commit().ensure();
+  vm::CellStorer stor{*cell_db_.get()};
   cell_db_->begin_write_batch().ensure();
   boc_->commit(stor).ensure();
   set_block(empty, std::move(E));
@@ -154,17 +133,10 @@ void CellDbIn::store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell,
   td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
 
   promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
-  release_db();
 }
 
 void CellDbIn::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) {
-  enqueue([this, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+  promise.set_result(boc_->get_cell_db_reader());
-    if (R.is_error()) {
-      return;
-    }
-    promise.set_result(boc_->get_cell_db_reader());
-    release_db();
-  });
 }
 
 void CellDbIn::alarm() {
@@ -214,30 +186,13 @@ void CellDbIn::gc_cont(BlockHandle handle) {
 }
 
 void CellDbIn::gc_cont2(BlockHandle handle) {
-  enqueue([this, handle = std::move(handle)](td::Result<td::Unit> R) mutable {
+  td::PerfWarningTimer timer{"gccell", 0.1};
-    if (R.is_error()) {
-      return;
-    }
-    td::Promise<td::Unit> promise = [timer = td::PerfWarningTimer{"gccell", 0.1}](td::Result<td::Unit>) {};
-    auto FR = get_block(get_key_hash(handle->id()));
-    FR.ensure();
-    auto F = FR.move_as_ok();
-    auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok();
 
-    boc_->dec(cell);
+  auto key_hash = get_key_hash(handle->id());
-    boc_->prepare_commit_async(async_executor, [SelfId = actor_id(this), promise = std::move(promise),
-                                                block_id = handle->id()](td::Result<td::Unit> R) mutable {
-      R.ensure();
-      td::actor::send_closure(SelfId, &CellDbIn::gc_cont3, block_id, std::move(promise));
-    });
-  });
-}
-
-void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise) {
-  auto key_hash = get_key_hash(block_id);
   auto FR = get_block(key_hash);
   FR.ensure();
   auto F = FR.move_as_ok();
+
   auto PR = get_block(F.prev);
   PR.ensure();
   auto P = PR.move_as_ok();
@@ -252,6 +207,10 @@ void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise) {
     N.next = N.prev;
   }
 
+  auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok();
+
+  boc_->dec(cell);
+  boc_->prepare_commit().ensure();
   vm::CellStorer stor{*cell_db_};
   cell_db_->begin_write_batch().ensure();
   boc_->commit(stor).ensure();
@@ -265,28 +224,6 @@ void CellDbIn::gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise) {
   td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot());
 
   DCHECK(get_block(key_hash).is_error());
-  promise.set_result(td::Unit());
-  release_db();
-}
-
-void CellDbIn::enqueue(td::Promise<td::Unit> promise) {
-  db_queue_.push(std::move(promise));
-  process_event();
-}
-
-void CellDbIn::release_db() {
-  db_busy_ = false;
-  process_event();
-}
-
-void CellDbIn::process_event() {
-  if (db_busy_ || db_queue_.empty()) {
-    return;
-  }
-  db_busy_ = true;
-  auto promise = std::move(db_queue_.front());
-  db_queue_.pop();
-  promise.set_result(td::Unit());
 }
 
 void CellDbIn::skip_gc() {

validator/db/celldb.hpp

@@ -25,7 +25,6 @@
 #include "ton/ton-types.h"
 #include "interfaces/block-handle.h"
 #include "auto/tl/ton_api.h"
-#include <queue>
 
 namespace ton {
 
@@ -88,11 +87,8 @@ class CellDbIn : public CellDbBase {
   void gc(BlockIdExt block_id);
   void gc_cont(BlockHandle handle);
   void gc_cont2(BlockHandle handle);
-  void gc_cont3(BlockIdExt block_id, td::Promise<td::Unit> promise);
   void skip_gc();
 
-  void store_cell_cont(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise);
-
   td::actor::ActorId<RootDb> root_db_;
   td::actor::ActorId<CellDb> parent_;
 
@@ -100,13 +96,6 @@ class CellDbIn : public CellDbBase {
 
   std::unique_ptr<vm::DynamicBagOfCellsDb> boc_;
   std::shared_ptr<vm::KeyValue> cell_db_;
-
-  std::queue<td::Promise<td::Unit>> db_queue_;
-  bool db_busy_ = false;
-
-  void enqueue(td::Promise<td::Unit> promise);
-  void release_db();
-  void process_event();
 };
 
 class CellDb : public CellDbBase {