updated tonlib, fixed bugs

updated tonlib
fixed bugs in func
validator: partial support for hardforks
liteserver: support for waitMasterchainBlock prefix
transactions: support for gas flat rate

validator/db/blockdb.cpp

@@ -229,6 +229,73 @@ tl_object_ptr<ton_api::db_blockdb_lru> BlockDb::DbEntry::release() {
   return create_tl_object<ton_api::db_blockdb_lru>(create_tl_block_id(block_id), prev, next);
 }
 
+void BlockDb::truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) {
+  std::map<ShardIdFull, BlockSeqno> max_seqno;
+  max_seqno.emplace(ShardIdFull{masterchainId}, state->get_seqno() + 1);
+
+  auto shards = state->get_shards();
+  auto it = KeyHash::zero();
+  kv_->begin_transaction().ensure();
+  while (true) {
+    auto R = get_block_lru(it);
+    R.ensure();
+    auto v = R.move_as_ok();
+    it = v.next;
+    R = get_block_lru(it);
+    R.ensure();
+    v = R.move_as_ok();
+    if (v.is_empty()) {
+      break;
+    }
+
+    auto s = v.block_id.shard_full();
+    if (!max_seqno.count(s)) {
+      bool found = false;
+      for (auto &shard : shards) {
+        if (shard_intersects(shard->shard(), s)) {
+          found = true;
+          max_seqno.emplace(s, shard->top_block_id().seqno() + 1);
+          break;
+        }
+      }
+      if (!found) {
+        max_seqno.emplace(s, 0);
+      }
+    }
+
+    bool to_delete = v.block_id.seqno() >= max_seqno[s];
+    if (to_delete) {
+      auto key_hash = get_block_value_key(v.block_id);
+      auto B = get_block_value(key_hash);
+      B.ensure();
+      auto handleR = create_block_handle(B.move_as_ok());
+      handleR.ensure();
+      auto handle = handleR.move_as_ok();
+
+      handle->unsafe_clear_applied();
+      handle->unsafe_clear_next();
+
+      if (handle->need_flush()) {
+        set_block_value(key_hash, handle->serialize());
+      }
+    } else if (v.block_id.seqno() + 1 == max_seqno[s]) {
+      auto key_hash = get_block_value_key(v.block_id);
+      auto B = get_block_value(key_hash);
+      B.ensure();
+      auto handleR = create_block_handle(B.move_as_ok());
+      handleR.ensure();
+      auto handle = handleR.move_as_ok();
+
+      handle->unsafe_clear_next();
+
+      if (handle->need_flush()) {
+        set_block_value(key_hash, handle->serialize());
+      }
+    }
+  }
+  kv_->commit_transaction().ensure();
+}
+
 }  // namespace validator
 
 }  // namespace ton

validator/db/blockdb.hpp

@@ -41,6 +41,8 @@ class BlockDb : public td::actor::Actor {
   void gc();
   void skip_gc();
 
+  void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise);
+
   BlockDb(td::actor::ActorId<RootDb> root_db, std::string db_path);
 
  private:

validator/db/filedb.cpp

@@ -138,7 +138,7 @@ void FileDb::load_file(RefId ref_id, td::Promise<td::BufferSlice> promise) {
         }
       });
 
-  td::actor::create_actor<db::ReadFile>("readfile", get_file_name(ref_id, false), 0, -1, std::move(P)).release();
+  td::actor::create_actor<db::ReadFile>("readfile", get_file_name(ref_id, false), 0, -1, 0, std::move(P)).release();
 }
 
 void FileDb::load_file_slice(RefId ref_id, td::int64 offset, td::int64 max_size, td::Promise<td::BufferSlice> promise) {
@@ -159,7 +159,7 @@ void FileDb::load_file_slice(RefId ref_id, td::int64 offset, td::int64 max_size,
     }
   });
 
-  td::actor::create_actor<db::ReadFile>("readfile", get_file_name(ref_id, false), offset, max_size, std::move(P))
+  td::actor::create_actor<db::ReadFile>("readfile", get_file_name(ref_id, false), offset, max_size, 0, std::move(P))
       .release();
 }
 

validator/db/files-async.hpp

@@ -81,25 +81,32 @@ class WriteFile : public td::actor::Actor {
 
 class ReadFile : public td::actor::Actor {
  public:
+  enum Flags : td::uint32 { f_disable_log = 1 };
   void start_up() override {
     auto S = td::read_file(file_name_, max_length_, offset_);
     if (S.is_ok()) {
       promise_.set_result(S.move_as_ok());
     } else {
       // TODO check error code
-      LOG(ERROR) << "missing file " << file_name_;
+      if (flags_ & Flags::f_disable_log) {
+        LOG(DEBUG) << "missing file " << file_name_;
+      } else {
+        LOG(ERROR) << "missing file " << file_name_;
+      }
       promise_.set_error(td::Status::Error(ErrorCode::notready, "file does not exist"));
     }
     stop();
   }
-  ReadFile(std::string file_name, td::int64 offset, td::int64 max_length, td::Promise<td::BufferSlice> promise)
-      : file_name_(file_name), offset_(offset), max_length_(max_length), promise_(std::move(promise)) {
+  ReadFile(std::string file_name, td::int64 offset, td::int64 max_length, td::uint32 flags,
+           td::Promise<td::BufferSlice> promise)
+      : file_name_(file_name), offset_(offset), max_length_(max_length), flags_(flags), promise_(std::move(promise)) {
   }
 
  private:
   std::string file_name_;
   td::int64 offset_;
   td::int64 max_length_;
+  td::uint32 flags_;
   td::Promise<td::BufferSlice> promise_;
 };
 

validator/db/ltdb.cpp

@@ -216,6 +216,95 @@ void LtDb::start_up() {
   kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
 }
 
+void LtDb::truncate_workchain(ShardIdFull shard, td::Ref<MasterchainState> state) {
+  auto key = get_desc_key(shard);
+  std::string value;
+  auto R = kv_->get(key, value);
+  R.ensure();
+  CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok);
+  auto F = fetch_tl_object<ton_api::db_lt_desc_value>(td::BufferSlice{value}, true);
+  F.ensure();
+  auto f = F.move_as_ok();
+
+  auto shards = state->get_shards();
+  BlockSeqno seqno = 0;
+  if (shard.is_masterchain()) {
+    seqno = state->get_seqno();
+  } else {
+    for (auto s : shards) {
+      if (shard_intersects(s->shard(), shard)) {
+        seqno = s->top_block_id().seqno();
+        break;
+      }
+    }
+  }
+
+  while (f->last_idx_ > f->first_idx_) {
+    auto db_key = get_el_key(shard, f->last_idx_ - 1);
+    R = kv_->get(db_key, value);
+    R.ensure();
+    CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok);
+    auto E = fetch_tl_object<ton_api::db_lt_el_value>(td::BufferSlice{value}, true);
+    E.ensure();
+    auto e = E.move_as_ok();
+
+    bool to_delete = static_cast<td::uint32>(e->id_->seqno_) > seqno;
+
+    if (!to_delete) {
+      break;
+    } else {
+      f->last_idx_--;
+      kv_->erase(db_key).ensure();
+    }
+  }
+
+  if (f->first_idx_ == f->last_idx_) {
+    f->last_ts_ = 0;
+    f->last_lt_ = 0;
+    f->last_seqno_ = 0;
+  }
+
+  kv_->set(key, serialize_tl_object(f, true)).ensure();
+}
+
+void LtDb::truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) {
+  auto status_key = create_serialize_tl_object<ton_api::db_lt_status_key>();
+  td::Result<td::KeyValue::GetStatus> R;
+  td::uint32 total_shards = 0;
+  {
+    std::string value;
+    R = kv_->get(status_key.as_slice(), value);
+    R.ensure();
+    if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+      promise.set_value(td::Unit());
+      return;
+    }
+    auto F = fetch_tl_object<ton_api::db_lt_status_value>(value, true);
+    F.ensure();
+    auto f = F.move_as_ok();
+    total_shards = f->total_shards_;
+    if (total_shards == 0) {
+      promise.set_value(td::Unit());
+      return;
+    }
+  }
+  kv_->begin_transaction().ensure();
+  for (td::uint32 idx = 0; idx < total_shards; idx++) {
+    auto shard_key = create_serialize_tl_object<ton_api::db_lt_shard_key>(idx);
+    std::string value;
+    R = kv_->get(shard_key.as_slice(), value);
+    R.ensure();
+    CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok);
+    auto F = fetch_tl_object<ton_api::db_lt_shard_value>(value, true);
+    F.ensure();
+    auto f = F.move_as_ok();
+
+    truncate_workchain(ShardIdFull{f->workchain_, static_cast<td::uint64>(f->shard_)}, state);
+  }
+  kv_->commit_transaction().ensure();
+  promise.set_value(td::Unit());
+}
+
 }  // namespace validator
 
 }  // namespace ton

validator/db/ltdb.hpp

@@ -20,6 +20,7 @@
 
 #include "td/actor/actor.h"
 #include "td/db/KeyValueAsync.h"
+#include "validator/interfaces/db.h"
 
 #include "ton/ton-types.h"
 
@@ -42,6 +43,9 @@ class LtDb : public td::actor::Actor {
   void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise<BlockIdExt> promise);
   void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise<BlockIdExt> promise);
 
+  void truncate_workchain(ShardIdFull shard, td::Ref<MasterchainState> state);
+  void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise);
+
   void start_up() override;
 
   LtDb(td::actor::ActorId<RootDb> root_db, std::string db_path) : root_db_(root_db), db_path_(std::move(db_path)) {

validator/db/rootdb.cpp

@@ -25,6 +25,7 @@
 #include "td/utils/overloaded.h"
 #include "common/checksum.h"
 #include "validator/stats-merger.h"
+#include "td/actor/MultiPromise.h"
 
 namespace ton {
 
@@ -413,6 +414,14 @@ void RootDb::get_async_serializer_state(td::Promise<AsyncSerializerState> promis
   td::actor::send_closure(state_db_, &StateDb::get_async_serializer_state, std::move(promise));
 }
 
+void RootDb::update_hardforks(std::vector<BlockIdExt> blocks, td::Promise<td::Unit> promise) {
+  td::actor::send_closure(state_db_, &StateDb::update_hardforks, std::move(blocks), std::move(promise));
+}
+
+void RootDb::get_hardforks(td::Promise<std::vector<BlockIdExt>> promise) {
+  td::actor::send_closure(state_db_, &StateDb::get_hardforks, std::move(promise));
+}
+
 void RootDb::start_up() {
   cell_db_ = td::actor::create_actor<CellDb>("celldb", actor_id(this), root_path_ + "/celldb/");
   block_db_ = td::actor::create_actor<BlockDb>("blockdb", actor_id(this), root_path_ + "/blockdb/");
@@ -481,6 +490,15 @@ void RootDb::prepare_stats(td::Promise<std::vector<std::pair<std::string, std::s
   td::actor::send_closure(archive_db_, &FileDb::prepare_stats, merger.make_promise("archivedb."));
 }
 
+void RootDb::truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) {
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  ig.add_promise(std::move(promise));
+
+  td::actor::send_closure(lt_db_, &LtDb::truncate, state, ig.get_promise());
+  td::actor::send_closure(block_db_, &BlockDb::truncate, state, ig.get_promise());
+}
+
 }  // namespace validator
 
 }  // namespace ton

validator/db/rootdb.hpp

@@ -104,6 +104,9 @@ class RootDb : public Db {
   void update_async_serializer_state(AsyncSerializerState state, td::Promise<td::Unit> promise) override;
   void get_async_serializer_state(td::Promise<AsyncSerializerState> promise) override;
 
+  void update_hardforks(std::vector<BlockIdExt> blocks, td::Promise<td::Unit> promise) override;
+  void get_hardforks(td::Promise<std::vector<BlockIdExt>> promise) override;
+
   void archive(BlockIdExt block_id, td::Promise<td::Unit> promise) override;
 
   void allow_state_gc(BlockIdExt block_id, td::Promise<bool> promise);
@@ -112,6 +115,8 @@ class RootDb : public Db {
 
   void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) override;
 
+  void truncate(td::Ref<MasterchainState> state, td::Promise<td::Unit> promise) override;
+
  private:
   td::actor::ActorId<ValidatorManager> validator_manager_;
 

validator/db/statedb.cpp

@@ -179,6 +179,44 @@ void StateDb::get_async_serializer_state(td::Promise<AsyncSerializerState> promi
                                          static_cast<UnixTime>(obj->last_ts_)});
 }
 
+void StateDb::update_hardforks(std::vector<BlockIdExt> blocks, td::Promise<td::Unit> promise) {
+  auto key = create_hash_tl_object<ton_api::db_state_key_hardforks>();
+
+  std::vector<tl_object_ptr<ton_api::tonNode_blockIdExt>> vec;
+
+  for (auto &e : blocks) {
+    vec.push_back(create_tl_block_id(e));
+  }
+
+  kv_->begin_transaction().ensure();
+  kv_->set(key.as_slice(), create_serialize_tl_object<ton_api::db_state_hardforks>(std::move(vec))).ensure();
+  kv_->commit_transaction();
+
+  promise.set_value(td::Unit());
+}
+
+void StateDb::get_hardforks(td::Promise<std::vector<BlockIdExt>> promise) {
+  auto key = create_hash_tl_object<ton_api::db_state_key_hardforks>();
+
+  std::string value;
+  auto R = kv_->get(key.as_slice(), value);
+  R.ensure();
+  if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) {
+    promise.set_value(std::vector<BlockIdExt>{});
+    return;
+  }
+  auto F = fetch_tl_object<ton_api::db_state_hardforks>(value, true);
+  F.ensure();
+  auto f = F.move_as_ok();
+
+  std::vector<BlockIdExt> vec;
+  for (auto &e : f->blocks_) {
+    vec.push_back(create_block_id(e));
+  }
+
+  promise.set_value(std::move(vec));
+}
+
 StateDb::StateDb(td::actor::ActorId<RootDb> root_db, std::string db_path) : root_db_(root_db), db_path_(db_path) {
 }
 

validator/db/statedb.hpp

@@ -47,6 +47,9 @@ class StateDb : public td::actor::Actor {
   void update_async_serializer_state(AsyncSerializerState state, td::Promise<td::Unit> promise);
   void get_async_serializer_state(td::Promise<AsyncSerializerState> promise);
 
+  void update_hardforks(std::vector<BlockIdExt> blocks, td::Promise<td::Unit> promise);
+  void get_hardforks(td::Promise<std::vector<BlockIdExt>> promise);
+
   StateDb(td::actor::ActorId<RootDb> root_db, std::string path);
 
   void start_up() override;

validator/db/staticfilesdb.cpp

@@ -25,7 +25,9 @@ namespace validator {
 
 void StaticFilesDb::load_file(FileHash file_hash, td::Promise<td::BufferSlice> promise) {
   auto path = path_ + "/" + file_hash.to_hex();
-  td::actor::create_actor<db::ReadFile>("read file", path, 0, -1, std::move(promise)).release();
+  td::actor::create_actor<db::ReadFile>("read file", path, 0, -1, db::ReadFile::Flags::f_disable_log,
+                                        std::move(promise))
+      .release();
 }
 
 }  // namespace validator