initial commit

crypto/vm/db/BlobView.cpp

@@ -0,0 +1,181 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "vm/db/BlobView.h"
+
+#include "td/utils/port/FileFd.h"
+#include "td/utils/HashMap.h"
+
+#include "td/utils/format.h"
+#include "td/utils/port/RwMutex.h"
+#include "td/utils/port/MemoryMapping.h"
+
+#include <limits>
+#include <mutex>
+
+namespace vm {
+td::Result<td::Slice> BlobView::view(td::MutableSlice slice, td::uint64 offset) {
+  if (offset > size() || slice.size() > size() - offset) {
+    return td::Status::Error(PSLICE() << "BlobView: invalid range requested " << td::tag("slice offset", offset)
+                                      << td::tag("slice size", slice.size()) << td::tag("blob size", size()));
+  }
+  return view_impl(slice, offset);
+}
+namespace {
+class BufferSliceBlobViewImpl : public BlobView {
+ public:
+  BufferSliceBlobViewImpl(td::BufferSlice slice) : slice_(std::move(slice)) {
+  }
+  td::Result<td::Slice> view_impl(td::MutableSlice slice, td::uint64 offset) override {
+    // optimize anyway
+    if (offset > std::numeric_limits<std::size_t>::max()) {
+      return td::Slice();
+    }
+    return slice_.as_slice().substr(static_cast<std::size_t>(offset), slice.size());
+  }
+  td::uint64 size() override {
+    return slice_.size();
+  }
+
+ private:
+  td::BufferSlice slice_;
+};
+}  // namespace
+
+std::unique_ptr<BlobView> BufferSliceBlobView::create(td::BufferSlice slice) {
+  return std::make_unique<BufferSliceBlobViewImpl>(std::move(slice));
+}
+
+class FileBlobViewImpl : public BlobView {
+ public:
+  FileBlobViewImpl(td::FileFd fd, td::uint64 file_size) : fd_(std::move(fd)), file_size_(file_size) {
+  }
+
+  td::uint64 size() override {
+    return file_size_;
+  }
+  td::Result<td::Slice> view_impl(td::MutableSlice slice, td::uint64 offset) override {
+    CHECK(offset < size());
+    CHECK(size() - offset >= slice.size());
+    slice.truncate(file_size_ - offset);
+    auto first_page = offset / page_size;
+    auto last_page = (offset + slice.size() - 1) / page_size;
+    td::uint64 res_offset = 0;
+    for (auto page_i = first_page; page_i <= last_page; page_i++) {
+      auto page_offset = page_i * page_size;
+      auto from = td::max(page_offset, offset);
+      auto till = td::min(page_offset + page_size, offset + slice.size());
+      CHECK(from < till);
+      TRY_RESULT(page, load_page(page_i));
+      auto len = till - from;
+      slice.substr(res_offset, len).copy_from(page.substr(from - page_offset, len));
+      res_offset += len;
+    }
+    CHECK(slice.size() == res_offset);
+    total_view_size_ += slice.size();
+    return slice;
+  }
+  ~FileBlobViewImpl() {
+    //LOG(ERROR) << "LOADED " << pages_.size() << " " << total_view_size_;
+  }
+
+ private:
+  td::FileFd fd_;
+  td::uint64 file_size_;
+  const td::uint64 page_size = 4096;
+  td::uint64 total_view_size_{0};
+
+  td::RwMutex pages_rw_mutex_;
+  td::HashMap<td::uint64, td::BufferSlice> pages_;
+
+  std::mutex fd_mutex_;
+
+  td::Result<td::Slice> load_page(td::uint64 page_i) {
+    {
+      auto pages_guard = pages_rw_mutex_.lock_read();
+      auto it = pages_.find(page_i);
+      if (it != pages_.end()) {
+        return it->second.as_slice();
+      }
+    }
+
+    std::lock_guard<std::mutex> fd_guard(fd_mutex_);
+    {
+      auto pages_guard = pages_rw_mutex_.lock_read();
+      auto it = pages_.find(page_i);
+      if (it != pages_.end()) {
+        return it->second.as_slice();
+      }
+    }
+    auto offset = page_i * page_size;
+
+    auto size = td::min(file_size_ - offset, page_size);
+    auto buffer_slice = td::BufferSlice(size);
+    TRY_RESULT(read_size, fd_.pread(buffer_slice.as_slice(), offset));
+    if (read_size != buffer_slice.size()) {
+      return td::Status::Error("not enough data in file");
+    }
+
+    auto pages_guard = pages_rw_mutex_.lock_write();
+    auto &res = pages_[page_i];
+    res = std::move(buffer_slice);
+    return res.as_slice();
+  }
+};
+td::Result<std::unique_ptr<BlobView>> FileBlobView::create(td::CSlice file_path, td::uint64 file_size) {
+  TRY_RESULT(fd, td::FileFd::open(file_path, td::FileFd::Flags::Read));
+  TRY_RESULT(stat, fd.stat());
+  if (file_size == 0) {
+    file_size = stat.size_;
+  } else if (file_size != (td::uint64)stat.size_) {
+    return td::Status::Error("Wrong file size");
+  }
+  return std::make_unique<FileBlobViewImpl>(std::move(fd), file_size);
+}
+
+class FileMemoryMappingBlobViewImpl : public BlobView {
+ public:
+  FileMemoryMappingBlobViewImpl(td::MemoryMapping mapping) : mapping_(std::move(mapping)) {
+  }
+  td::Result<td::Slice> view_impl(td::MutableSlice slice, td::uint64 offset) override {
+    // optimize anyway
+    return mapping_.as_slice().substr(offset, slice.size());
+  }
+  td::uint64 size() override {
+    return mapping_.as_slice().size();
+  }
+
+ private:
+  td::MemoryMapping mapping_;
+};
+
+td::Result<std::unique_ptr<BlobView>> FileMemoryMappingBlobView::create(td::CSlice file_path, td::uint64 file_size) {
+  TRY_RESULT(fd, td::FileFd::open(file_path, td::FileFd::Flags::Read));
+  TRY_RESULT(stat, fd.stat());
+  if (file_size == 0) {
+    file_size = stat.size_;
+  } else if (file_size != (td::uint64)stat.size_) {
+    return td::Status::Error("Wrong file size");
+  }
+
+  TRY_RESULT(mapping, td::MemoryMapping::create_from_file(fd));
+
+  return std::make_unique<FileMemoryMappingBlobViewImpl>(std::move(mapping));
+}
+
+}  // namespace vm

crypto/vm/db/BlobView.h

@@ -0,0 +1,45 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "td/utils/buffer.h"
+
+namespace vm {
+class BlobView {
+ public:
+  virtual ~BlobView() = default;
+  td::Result<td::Slice> view(td::MutableSlice slice, td::uint64 offset);
+  virtual td::uint64 size() = 0;
+
+ private:
+  virtual td::Result<td::Slice> view_impl(td::MutableSlice slice, td::uint64 offset) = 0;
+};
+
+class BufferSliceBlobView {
+ public:
+  static std::unique_ptr<BlobView> create(td::BufferSlice slice);
+};
+class FileBlobView {
+ public:
+  static td::Result<std::unique_ptr<BlobView>> create(td::CSlice file_path, td::uint64 file_size = 0);
+};
+class FileMemoryMappingBlobView {
+ public:
+  static td::Result<std::unique_ptr<BlobView>> create(td::CSlice file_path, td::uint64 file_size = 0);
+};
+}  // namespace vm

crypto/vm/db/CellHashTable.h

@@ -0,0 +1,71 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "td/utils/Slice.h"
+
+#include <set>
+
+namespace vm {
+template <class InfoT>
+class CellHashTable {
+ public:
+  template <class F>
+  InfoT &apply(td::Slice hash, F &&f) {
+    auto it = set_.find(hash);
+    if (it != set_.end()) {
+      auto &res = const_cast<InfoT &>(*it);
+      f(res);
+      return res;
+    }
+    InfoT info;
+    f(info);
+    auto &res = const_cast<InfoT &>(*(set_.insert(std::move(info)).first));
+    return res;
+  }
+
+  template <class F>
+  void for_each(F &&f) {
+    for (auto &info : set_) {
+      f(info);
+    }
+  }
+  template <class F>
+  void filter(F &&f) {
+    for (auto it = set_.begin(); it != set_.end();) {
+      if (f(*it)) {
+        it++;
+      } else {
+        it = set_.erase(it);
+      }
+    }
+  }
+  void erase(td::Slice hash) {
+    auto it = set_.find(hash);
+    CHECK(it != set_.end());
+    set_.erase(it);
+  }
+  size_t size() const {
+    return set_.size();
+  }
+
+ private:
+  std::set<InfoT, std::less<>> set_;
+};
+}  // namespace vm

crypto/vm/db/CellStorage.cpp

@@ -0,0 +1,163 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "vm/db/CellStorage.h"
+#include "vm/db/DynamicBagOfCellsDb.h"
+#include "vm/boc.h"
+#include "td/utils/base64.h"
+#include "td/utils/tl_parsers.h"
+#include "td/utils/tl_helpers.h"
+
+namespace vm {
+namespace {
+class RefcntCellStorer {
+ public:
+  RefcntCellStorer(td::int32 refcnt, const DataCell &cell) : refcnt_(refcnt), cell_(cell) {
+  }
+
+  template <class StorerT>
+  void store(StorerT &storer) const {
+    using td::store;
+    store(refcnt_, storer);
+    store(cell_, storer);
+    for (unsigned i = 0; i < cell_.size_refs(); i++) {
+      auto cell = cell_.get_ref(i);
+      auto level_mask = cell->get_level_mask();
+      auto level = level_mask.get_level();
+      td::uint8 x = static_cast<td::uint8>(level_mask.get_mask());
+      storer.store_slice(td::Slice(&x, 1));
+      for (unsigned level_i = 0; level_i <= level; level_i++) {
+        if (!level_mask.is_significant(level_i)) {
+          continue;
+        }
+        storer.store_slice(cell->get_hash(level_i).as_slice());
+      }
+      for (unsigned level_i = 0; level_i <= level; level_i++) {
+        if (!level_mask.is_significant(level_i)) {
+          continue;
+        }
+        td::uint8 depth_buf[Cell::depth_bytes];
+        DataCell::store_depth(depth_buf, cell->get_depth(level_i));
+        storer.store_slice(td::Slice(depth_buf, Cell::depth_bytes));
+      }
+    }
+  }
+
+ private:
+  td::int32 refcnt_;
+  const DataCell &cell_;
+};
+
+class RefcntCellParser {
+ public:
+  RefcntCellParser(bool need_data) : need_data_(need_data) {
+  }
+  td::int32 refcnt;
+  Ref<DataCell> cell;
+
+  template <class ParserT>
+  void parse(ParserT &parser, ExtCellCreator &ext_cell_creator) {
+    using ::td::parse;
+    parse(refcnt, parser);
+    if (!need_data_) {
+      return;
+    }
+    auto status = [&]() -> td::Status {
+      TRY_STATUS(parser.get_status());
+      auto size = parser.get_left_len();
+      td::Slice data = parser.template fetch_string_raw<td::Slice>(size);
+      CellSerializationInfo info;
+      auto cell_data = data;
+      TRY_STATUS(info.init(cell_data, 0 /*ref_byte_size*/));
+      data = data.substr(info.end_offset);
+
+      Ref<Cell> refs[Cell::max_refs];
+      for (int i = 0; i < info.refs_cnt; i++) {
+        if (data.size() < 1) {
+          return td::Status::Error("Not enought data");
+        }
+        Cell::LevelMask level_mask(data[0]);
+        auto n = level_mask.get_hashes_count();
+        auto end_offset = 1 + n * (Cell::hash_bytes + Cell::depth_bytes);
+        if (data.size() < end_offset) {
+          return td::Status::Error("Not enought data");
+        }
+
+        TRY_RESULT(ext_cell, ext_cell_creator.ext_cell(level_mask, data.substr(1, n * Cell::hash_bytes),
+                                                       data.substr(1 + n * Cell::hash_bytes, n * Cell::depth_bytes)));
+        refs[i] = std::move(ext_cell);
+        CHECK(refs[i]->get_level() == level_mask.get_level());
+        data = data.substr(end_offset);
+      }
+      if (!data.empty()) {
+        return td::Status::Error("Too much data");
+      }
+      TRY_RESULT(data_cell, info.create_data_cell(cell_data, td::Span<Ref<Cell>>(refs, info.refs_cnt)));
+      cell = std::move(data_cell);
+      return td::Status::OK();
+    }();
+    if (status.is_error()) {
+      parser.set_error(status.message().str());
+      return;
+    }
+  }
+
+ private:
+  bool need_data_;
+};
+}  // namespace
+
+CellLoader::CellLoader(std::shared_ptr<KeyValueReader> reader) : reader_(std::move(reader)) {
+  CHECK(reader_);
+}
+
+td::Result<CellLoader::LoadResult> CellLoader::load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator) {
+  //LOG(ERROR) << "Storage: load cell " << hash.size() << " " << td::base64_encode(hash);
+  LoadResult res;
+  std::string serialized;
+  TRY_RESULT(get_status, reader_->get(hash, serialized));
+  if (get_status != KeyValue::GetStatus::Ok) {
+    DCHECK(get_status == KeyValue::GetStatus::NotFound);
+    return res;
+  }
+
+  res.status = LoadResult::Ok;
+
+  RefcntCellParser refcnt_cell(need_data);
+  td::TlParser parser(serialized);
+  refcnt_cell.parse(parser, ext_cell_creator);
+  TRY_STATUS(parser.get_status());
+
+  res.refcnt_ = refcnt_cell.refcnt;
+  res.cell_ = std::move(refcnt_cell.cell);
+  //CHECK(res.cell_->get_hash() == hash);
+
+  return res;
+}
+
+CellStorer::CellStorer(KeyValue &kv) : kv_(kv) {
+}
+
+td::Status CellStorer::erase(td::Slice hash) {
+  return kv_.erase(hash);
+}
+
+td::Status CellStorer::set(td::int32 refcnt, const DataCell &cell) {
+  return kv_.set(cell.get_hash().as_slice(), td::serialize(RefcntCellStorer(refcnt, cell)));
+}
+}  // namespace vm

crypto/vm/db/CellStorage.h

@@ -0,0 +1,65 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "td/db/KeyValue.h"
+#include "vm/db/DynamicBagOfCellsDb.h"
+#include "vm/cells.h"
+
+#include "td/utils/Slice.h"
+#include "td/utils/Status.h"
+
+namespace vm {
+using KeyValue = td::KeyValue;
+using KeyValueReader = td::KeyValueReader;
+
+class CellLoader {
+ public:
+  struct LoadResult {
+   public:
+    enum { Ok, NotFound } status{NotFound};
+
+    Ref<DataCell> &cell() {
+      DCHECK(status == Ok);
+      return cell_;
+    }
+
+    td::int32 refcnt() const {
+      return refcnt_;
+    }
+
+    Ref<DataCell> cell_;
+    td::int32 refcnt_{0};
+  };
+  CellLoader(std::shared_ptr<KeyValueReader> reader);
+  td::Result<LoadResult> load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator);
+
+ private:
+  std::shared_ptr<KeyValueReader> reader_;
+};
+
+class CellStorer {
+ public:
+  CellStorer(KeyValue &kv);
+  td::Status erase(td::Slice hash);
+  td::Status set(td::int32 refcnt, const DataCell &cell);
+
+ private:
+  KeyValue &kv_;
+};
+}  // namespace vm

crypto/vm/db/DynamicBagOfCellsDb.cpp

@@ -0,0 +1,500 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "vm/db/DynamicBagOfCellsDb.h"
+#include "vm/db/CellStorage.h"
+#include "vm/db/CellHashTable.h"
+
+#include "vm/cells/ExtCell.h"
+
+#include "td/utils/base64.h"
+#include "td/utils/format.h"
+#include "td/utils/ThreadSafeCounter.h"
+
+#include "vm/cellslice.h"
+
+namespace vm {
+namespace {
+
+class CellDbReader {
+ public:
+  virtual ~CellDbReader() = default;
+  virtual td::Result<Ref<DataCell>> load_cell(td::Slice hash) = 0;
+};
+
+struct DynamicBocExtCellExtra {
+  std::shared_ptr<CellDbReader> reader;
+};
+
+class DynamicBocCellLoader {
+ public:
+  static td::Result<Ref<DataCell>> load_data_cell(const Cell &cell, const DynamicBocExtCellExtra &extra) {
+    return extra.reader->load_cell(cell.get_hash().as_slice());
+  }
+};
+
+using DynamicBocExtCell = ExtCell<DynamicBocExtCellExtra, DynamicBocCellLoader>;
+
+struct CellInfo {
+  bool sync_with_db{false};
+  bool in_db{false};
+
+  bool was_dfs_new_cells{false};
+  bool was{false};
+
+  td::int32 db_refcnt{0};
+  td::int32 refcnt_diff{0};
+  Ref<Cell> cell;
+  Cell::Hash key() const {
+    return cell->get_hash();
+  }
+  bool operator<(const CellInfo &other) const {
+    return key() < other.key();
+  }
+};
+
+bool operator<(const CellInfo &a, td::Slice b) {
+  return a.key().as_slice() < b;
+}
+
+bool operator<(td::Slice a, const CellInfo &b) {
+  return a < b.key().as_slice();
+}
+
+class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreator {
+ public:
+  DynamicBagOfCellsDbImpl() {
+    get_thread_safe_counter().add(1);
+  }
+  ~DynamicBagOfCellsDbImpl() {
+    get_thread_safe_counter().add(-1);
+    reset_cell_db_reader();
+  }
+  td::Result<Ref<Cell>> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override {
+    return get_cell_info_lazy(level_mask, hash, depth).cell;
+  }
+  td::Result<Ref<DataCell>> load_cell(td::Slice hash) override {
+    TRY_RESULT(loaded_cell, get_cell_info_force(hash).cell->load_cell());
+    return std::move(loaded_cell.data_cell);
+  }
+  CellInfo &get_cell_info_force(td::Slice hash) {
+    return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); });
+  }
+  CellInfo &get_cell_info_lazy(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) {
+    return hash_table_.apply(hash.substr(hash.size() - Cell::hash_bytes),
+                             [&](CellInfo &info) { update_cell_info_lazy(info, level_mask, hash, depth); });
+  }
+  CellInfo &get_cell_info(const Ref<Cell> &cell) {
+    return hash_table_.apply(cell->get_hash().as_slice(), [&](CellInfo &info) { update_cell_info(info, cell); });
+  }
+
+  void inc(const Ref<Cell> &cell) override {
+    if (cell.is_null()) {
+      return;
+    }
+    if (cell->get_virtualization() != 0) {
+      return;
+    }
+    //LOG(ERROR) << "INC";
+    //CellSlice(cell, nullptr).print_rec(std::cout);
+    to_inc_.push_back(cell);
+  }
+  void dec(const Ref<Cell> &cell) override {
+    if (cell.is_null()) {
+      return;
+    }
+    if (cell->get_virtualization() != 0) {
+      return;
+    }
+    //LOG(ERROR) << "DEC";
+    //CellSlice(cell, nullptr).print_rec(std::cout);
+    to_dec_.push_back(cell);
+  }
+
+  bool is_prepared_for_commit() {
+    return to_inc_.empty() && to_dec_.empty();
+  }
+
+  Stats get_stats_diff() override {
+    CHECK(is_prepared_for_commit());
+    return stats_diff_;
+  }
+
+  td::Status prepare_commit() override {
+    if (is_prepared_for_commit()) {
+      return td::Status::OK();
+    }
+    //LOG(ERROR) << "dfs_new_cells_in_db";
+    for (auto &new_cell : to_inc_) {
+      auto &new_cell_info = get_cell_info(new_cell);
+      dfs_new_cells_in_db(new_cell_info);
+    }
+    //return td::Status::OK();
+    //LOG(ERROR) << "dfs_new_cells";
+    for (auto &new_cell : to_inc_) {
+      auto &new_cell_info = get_cell_info(new_cell);
+      dfs_new_cells(new_cell_info);
+    }
+
+    //LOG(ERROR) << "dfs_old_cells";
+    for (auto &old_cell : to_dec_) {
+      auto &old_cell_info = get_cell_info(old_cell);
+      dfs_old_cells(old_cell_info);
+    }
+
+    //LOG(ERROR) << "save_diff_prepare";
+    save_diff_prepare();
+
+    to_inc_.clear();
+    to_dec_.clear();
+
+    return td::Status::OK();
+  }
+
+  td::Status commit(CellStorer &storer) override {
+    prepare_commit();
+    save_diff(storer);
+    // Some elements are erased from hash table, to keep it small.
+    // Hash table is no longer represents the difference between the loader and
+    // the current bag of cells.
+    reset_cell_db_reader();
+    return td::Status::OK();
+  }
+
+  td::Status set_loader(std::unique_ptr<CellLoader> loader) override {
+    reset_cell_db_reader();
+    loader_ = std::move(loader);
+    //cell_db_reader_ = std::make_shared<CellDbReaderImpl>(this);
+    // Temporary(?) fix to make ExtCell thread safe.
+    // Downside(?) - loaded cells won't be cached
+    cell_db_reader_ = std::make_shared<CellDbReaderImpl>(std::make_unique<CellLoader>(*loader_));
+    stats_diff_ = {};
+    return td::Status::OK();
+  }
+
+ private:
+  std::unique_ptr<CellLoader> loader_;
+  std::vector<Ref<Cell>> to_inc_;
+  std::vector<Ref<Cell>> to_dec_;
+  CellHashTable<CellInfo> hash_table_;
+  std::vector<CellInfo *> visited_;
+  Stats stats_diff_;
+
+  static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() {
+    static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDb");
+    return res;
+  }
+
+  class CellDbReaderImpl : public CellDbReader,
+                           private ExtCellCreator,
+                           public std::enable_shared_from_this<CellDbReaderImpl> {
+   public:
+    CellDbReaderImpl(std::unique_ptr<CellLoader> cell_loader) : db_(nullptr), cell_loader_(std::move(cell_loader)) {
+      if (cell_loader_) {
+        get_thread_safe_counter().add(1);
+      }
+    }
+    CellDbReaderImpl(DynamicBagOfCellsDb *db) : db_(db) {
+    }
+    ~CellDbReaderImpl() {
+      if (cell_loader_) {
+        get_thread_safe_counter().add(-1);
+      }
+    }
+    void set_loader(std::unique_ptr<CellLoader> cell_loader) {
+      if (cell_loader_) {
+        // avoid race
+        return;
+      }
+      cell_loader_ = std::move(cell_loader);
+      db_ = nullptr;
+      if (cell_loader_) {
+        get_thread_safe_counter().add(1);
+      }
+    }
+
+    td::Result<Ref<Cell>> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override {
+      CHECK(!db_);
+      TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth},
+                                                     DynamicBocExtCellExtra{shared_from_this()}));
+      return std::move(ext_cell);
+    }
+
+    td::Result<Ref<DataCell>> load_cell(td::Slice hash) override {
+      if (db_) {
+        return db_->load_cell(hash);
+      }
+      TRY_RESULT(load_result, cell_loader_->load(hash, true, *this));
+      CHECK(load_result.status == CellLoader::LoadResult::Ok);
+      return std::move(load_result.cell());
+    }
+
+   private:
+    static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() {
+      static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDbLoader");
+      return res;
+    }
+    DynamicBagOfCellsDb *db_;
+    std::unique_ptr<CellLoader> cell_loader_;
+  };
+
+  std::shared_ptr<CellDbReaderImpl> cell_db_reader_;
+
+  void reset_cell_db_reader() {
+    if (!cell_db_reader_) {
+      return;
+    }
+    cell_db_reader_->set_loader(std::move(loader_));
+    cell_db_reader_.reset();
+    //EXPERIMENTAL: clear cache to drop all references to old reader.
+    hash_table_ = {};
+  }
+
+  bool is_in_db(CellInfo &info) {
+    if (info.in_db) {
+      return true;
+    }
+    load_cell(info);
+    return info.in_db;
+  }
+  bool is_loaded(CellInfo &info) {
+    return info.sync_with_db;
+  }
+
+  void load_cell(CellInfo &info) {
+    if (is_loaded(info)) {
+      return;
+    }
+    do_load_cell(info);
+  }
+
+  bool dfs_new_cells_in_db(CellInfo &info) {
+    if (info.sync_with_db) {
+      return is_in_db(info);
+    }
+    if (info.in_db) {
+      return true;
+    }
+
+    bool not_in_db = false;
+    for_each(
+        info, [&not_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false);
+
+    if (not_in_db) {
+      CHECK(!info.in_db);
+      info.sync_with_db = true;
+    }
+    return is_in_db(info);
+  }
+
+  void dfs_new_cells(CellInfo &info) {
+    info.refcnt_diff++;
+    if (!info.was) {
+      info.was = true;
+      visited_.push_back(&info);
+    }
+    //LOG(ERROR) << "dfs new " << td::format::escaped(info.cell->hash());
+
+    if (info.was_dfs_new_cells) {
+      return;
+    }
+    info.was_dfs_new_cells = true;
+
+    if (is_in_db(info)) {
+      return;
+    }
+
+    CHECK(is_loaded(info));
+    for_each(info, [this](auto &child_info) { dfs_new_cells(child_info); });
+  }
+
+  void dfs_old_cells(CellInfo &info) {
+    info.refcnt_diff--;
+    if (!info.was) {
+      info.was = true;
+      visited_.push_back(&info);
+    }
+    //LOG(ERROR) << "dfs old " << td::format::escaped(info.cell->hash());
+
+    load_cell(info);
+
+    auto new_refcnt = info.refcnt_diff + info.db_refcnt;
+    CHECK(new_refcnt >= 0);
+    if (new_refcnt != 0) {
+      return;
+    }
+
+    for_each(info, [this](auto &child_info) { dfs_old_cells(child_info); });
+  }
+
+  void save_diff_prepare() {
+    stats_diff_ = {};
+    for (auto info_ptr : visited_) {
+      save_cell_prepare(*info_ptr);
+    }
+  }
+
+  void save_diff(CellStorer &storer) {
+    //LOG(ERROR) << hash_table_.size();
+    for (auto info_ptr : visited_) {
+      save_cell(*info_ptr, storer);
+    }
+    visited_.clear();
+  }
+
+  void save_cell_prepare(CellInfo &info) {
+    if (info.refcnt_diff == 0) {
+      //CellSlice(info.cell, nullptr).print_rec(std::cout);
+      return;
+    }
+    load_cell(info);
+
+    auto loaded_cell = info.cell->load_cell().move_as_ok();
+    if (info.db_refcnt + info.refcnt_diff == 0) {
+      CHECK(info.in_db);
+      // erase
+      stats_diff_.cells_total_count--;
+      stats_diff_.cells_total_size -= loaded_cell.data_cell->get_serialized_size(true);
+    } else {
+      //save
+      if (info.in_db == false) {
+        stats_diff_.cells_total_count++;
+        stats_diff_.cells_total_size += loaded_cell.data_cell->get_serialized_size(true);
+      }
+    }
+  }
+
+  void save_cell(CellInfo &info, CellStorer &storer) {
+    auto guard = td::ScopeExit{} + [&] {
+      info.was_dfs_new_cells = false;
+      info.was = false;
+    };
+    if (info.refcnt_diff == 0) {
+      //CellSlice(info.cell, nullptr).print_rec(std::cout);
+      return;
+    }
+    CHECK(info.sync_with_db);
+
+    info.db_refcnt += info.refcnt_diff;
+    info.refcnt_diff = 0;
+
+    if (info.db_refcnt == 0) {
+      CHECK(info.in_db);
+      //LOG(ERROR) << "ERASE";
+      //CellSlice(NoVm(), info.cell).print_rec(std::cout);
+      storer.erase(info.cell->get_hash().as_slice());
+      info.in_db = false;
+      hash_table_.erase(info.cell->get_hash().as_slice());
+      guard.dismiss();
+    } else {
+      //LOG(ERROR) << "SAVE " << info.db_refcnt;
+      //CellSlice(NoVm(), info.cell).print_rec(std::cout);
+      auto loaded_cell = info.cell->load_cell().move_as_ok();
+      storer.set(info.db_refcnt, *loaded_cell.data_cell);
+      info.in_db = true;
+    }
+  }
+
+  template <class F>
+  void for_each(CellInfo &info, F &&f, bool force = true) {
+    auto cell = info.cell;
+
+    if (!cell->is_loaded()) {
+      if (!force) {
+        return;
+      }
+      load_cell(info);
+      cell = info.cell;
+    }
+    if (!cell->is_loaded()) {
+      cell->load_cell().ensure();
+    }
+    CHECK(cell->is_loaded());
+    vm::CellSlice cs(vm::NoVm{}, cell);  // FIXME
+    for (unsigned i = 0; i < cs.size_refs(); i++) {
+      //LOG(ERROR) << "---> " << td::format::escaped(cell->ref(i)->hash());
+      f(get_cell_info(cs.prefetch_ref(i)));
+    }
+  }
+
+  void do_load_cell(CellInfo &info) {
+    update_cell_info_force(info, info.cell->get_hash().as_slice());
+  }
+
+  void update_cell_info(CellInfo &info, const Ref<Cell> &cell) {
+    CHECK(!cell.is_null());
+    if (info.sync_with_db) {
+      return;
+    }
+    info.cell = cell;
+  }
+
+  void update_cell_info_lazy(CellInfo &info, Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) {
+    if (info.sync_with_db) {
+      CHECK(info.cell.not_null());
+      CHECK(info.cell->get_level_mask() == level_mask);
+      return;
+    }
+    if (info.cell.is_null()) {
+      auto ext_cell_r = create_empty_ext_cell(level_mask, hash, depth);
+      if (ext_cell_r.is_error()) {
+        //FIXME
+        LOG(ERROR) << "Failed to create ext_cell" << ext_cell_r.error();
+        return;
+      }
+      info.cell = ext_cell_r.move_as_ok();
+      info.in_db = true;  // TODO
+    }
+  }
+  void update_cell_info_force(CellInfo &info, td::Slice hash) {
+    if (info.sync_with_db) {
+      return;
+    }
+
+    do {
+      CHECK(loader_);
+      auto r_res = loader_->load(hash, true, *this);
+      if (r_res.is_error()) {
+        //FIXME
+        LOG(ERROR) << "Failed to load cell from db" << r_res.error();
+        break;
+      }
+      auto res = r_res.move_as_ok();
+      if (res.status != CellLoader::LoadResult::Ok) {
+        break;
+      }
+      info.cell = std::move(res.cell());
+      CHECK(info.cell->get_hash().as_slice() == hash);
+      info.in_db = true;
+      info.db_refcnt = res.refcnt();
+    } while (false);
+    info.sync_with_db = true;
+  }
+
+  td::Result<Ref<Cell>> create_empty_ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) {
+    TRY_RESULT(res, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth},
+                                              DynamicBocExtCellExtra{cell_db_reader_}));
+    return std::move(res);
+  }
+};
+}  // namespace
+
+std::unique_ptr<DynamicBagOfCellsDb> DynamicBagOfCellsDb::create() {
+  return std::make_unique<DynamicBagOfCellsDbImpl>();
+}
+}  // namespace vm

crypto/vm/db/DynamicBagOfCellsDb.h

@@ -0,0 +1,62 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "vm/cells.h"
+
+#include "td/utils/Slice.h"
+#include "td/utils/Status.h"
+
+namespace vm {
+class CellLoader;
+class CellStorer;
+}  // namespace vm
+
+namespace vm {
+class ExtCellCreator {
+ public:
+  virtual ~ExtCellCreator() = default;
+  virtual td::Result<Ref<Cell>> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) = 0;
+};
+
+class DynamicBagOfCellsDb {
+ public:
+  virtual ~DynamicBagOfCellsDb() = default;
+  virtual td::Result<Ref<DataCell>> load_cell(td::Slice hash) = 0;
+  struct Stats {
+    td::int64 cells_total_count{0};
+    td::int64 cells_total_size{0};
+    void apply_diff(Stats diff) {
+      cells_total_count += diff.cells_total_count;
+      cells_total_size += diff.cells_total_size;
+    }
+  };
+  virtual void inc(const Ref<Cell> &old_root) = 0;
+  virtual void dec(const Ref<Cell> &old_root) = 0;
+
+  virtual td::Status prepare_commit() = 0;
+  virtual Stats get_stats_diff() = 0;
+  virtual td::Status commit(CellStorer &) = 0;
+
+  // restart with new loader will also reset stats_diff
+  virtual td::Status set_loader(std::unique_ptr<CellLoader> loader) = 0;
+
+  static std::unique_ptr<DynamicBagOfCellsDb> create();
+};
+
+}  // namespace vm

crypto/vm/db/StaticBagOfCellsDb.cpp

@@ -0,0 +1,545 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "vm/db/StaticBagOfCellsDb.h"
+
+#include "vm/cells/CellWithStorage.h"
+#include "vm/boc.h"
+
+#include "vm/cells/ExtCell.h"
+
+#include "td/utils/crypto.h"
+#include "td/utils/format.h"
+#include "td/utils/misc.h"
+#include "td/utils/port/RwMutex.h"
+#include "td/utils/ConcurrentHashTable.h"
+
+#include <limits>
+
+namespace vm {
+//
+// Common interface
+//
+template <class ExtraT>
+class RootCell : public Cell {
+  struct PrivateTag {};
+
+ public:
+  td::Result<LoadedCell> load_cell() const override {
+    return cell_->load_cell();
+  }
+  Ref<Cell> virtualize(VirtualizationParameters virt) const override {
+    return cell_->virtualize(virt);
+  }
+  td::uint32 get_virtualization() const override {
+    return cell_->get_virtualization();
+  }
+  CellUsageTree::NodePtr get_tree_node() const override {
+    return cell_->get_tree_node();
+  }
+  bool is_loaded() const override {
+    return cell_->is_loaded();
+  }
+
+  // hash and level
+  LevelMask get_level_mask() const override {
+    return cell_->get_level_mask();
+  }
+
+  template <class T>
+  static Ref<Cell> create(Ref<Cell> cell, T&& extra) {
+    return Ref<RootCell>(true, std::move(cell), std::forward<T>(extra), PrivateTag{});
+  }
+  template <class T>
+  RootCell(Ref<Cell> cell, T&& extra, PrivateTag) : cell_(std::move(cell)), extra_(std::forward<T>(extra)) {
+  }
+
+ private:
+  Ref<Cell> cell_;
+  ExtraT extra_;
+  td::uint16 do_get_depth(td::uint32 level) const override {
+    return cell_->get_depth(level);
+  }
+  const Hash do_get_hash(td::uint32 level) const override {
+    return cell_->get_hash(level);
+  }
+};
+
+class DataCellCacheNoop {
+ public:
+  Ref<DataCell> store(int idx, Ref<DataCell> cell) {
+    return cell;
+  }
+  Ref<DataCell> load(int idx) {
+    return {};
+  }
+  void clear() {
+  }
+};
+class DataCellCacheMutex {
+ public:
+  Ref<DataCell> store(int idx, Ref<DataCell> cell) {
+    auto lock = cells_rw_mutex_.lock_write();
+    return cells_.emplace(idx, std::move(cell)).first->second;
+  }
+  Ref<DataCell> load(int idx) {
+    auto lock = cells_rw_mutex_.lock_read();
+    auto it = cells_.find(idx);
+    if (it != cells_.end()) {
+      return it->second;
+    }
+    return {};
+  }
+  void clear() {
+    auto guard = cells_rw_mutex_.lock_write();
+    cells_.clear();
+  }
+
+ private:
+  td::RwMutex cells_rw_mutex_;
+  td::HashMap<int, Ref<DataCell>> cells_;
+};
+
+class DataCellCacheTdlib {
+ public:
+  Ref<DataCell> store(int idx, Ref<DataCell> cell) {
+    return Ref<DataCell>(cells_.insert(as_key(idx), cell.get()));
+  }
+  Ref<DataCell> load(int idx) {
+    return Ref<DataCell>(cells_.find(as_key(idx), nullptr));
+  }
+
+  void clear() {
+    cells_.for_each([](auto key, auto value) { Ref<DataCell>{value}; });
+  }
+
+ private:
+  td::ConcurrentHashMap<td::uint32, const DataCell*> cells_;
+  td::uint32 as_key(int idx) {
+    td::uint32 key = static_cast<td::uint32>(idx + 1);
+    key *= 1000000007;
+    return key;
+  }
+};
+
+struct StaticBocExtCellExtra {
+  int idx;
+  std::weak_ptr<StaticBagOfCellsDb> deserializer;
+};
+
+class StaticBocLoader {
+ public:
+  static td::Result<Ref<DataCell>> load_data_cell(const Cell& cell, const StaticBocExtCellExtra& extra) {
+    auto deserializer = extra.deserializer.lock();
+    if (!deserializer) {
+      return td::Status::Error("StaticBocDb is already destroyed, cannot fetch cell");
+    }
+    return deserializer->load_by_idx(extra.idx);
+  }
+};
+using StaticBocExtCell = ExtCell<StaticBocExtCellExtra, StaticBocLoader>;
+using StaticBocRootCell = RootCell<std::shared_ptr<StaticBagOfCellsDb>>;
+
+td::Result<Ref<Cell>> StaticBagOfCellsDb::create_ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth,
+                                                          int idx) {
+  TRY_RESULT(res, StaticBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth},
+                                           StaticBocExtCellExtra{idx, shared_from_this()}));
+  return std::move(res);
+}
+
+//
+// Baseline implementation
+//
+class StaticBagOfCellsDbBaselineImpl : public StaticBagOfCellsDb {
+ public:
+  StaticBagOfCellsDbBaselineImpl(std::vector<Ref<Cell>> roots) : roots_(std::move(roots)) {
+  }
+  td::Result<size_t> get_root_count() override {
+    return roots_.size();
+  };
+  td::Result<Ref<Cell>> get_root_cell(size_t idx) override {
+    if (idx >= roots_.size()) {
+      return td::Status::Error(PSLICE() << "invalid root_cell index: " << idx);
+    }
+    return roots_[idx];
+  };
+
+ private:
+  std::vector<Ref<Cell>> roots_;
+
+  td::Result<Ref<DataCell>> load_by_idx(int idx) override {
+    UNREACHABLE();
+  }
+};
+
+td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbBaseline::create(std::unique_ptr<BlobView> data) {
+  std::string buf(data->size(), '\0');
+  TRY_RESULT(slice, data->view(buf, 0));
+  return create(slice);
+}
+
+td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbBaseline::create(td::Slice data) {
+  BagOfCells boc;
+  TRY_RESULT(x, boc.deserialize(data));
+  if (x <= 0) {
+    return td::Status::Error("failed to deserialize");
+  }
+  std::vector<Ref<Cell>> roots(boc.get_root_count());
+  for (int i = 0; i < boc.get_root_count(); i++) {
+    roots[i] = boc.get_root_cell(i);
+  }
+  return std::make_shared<StaticBagOfCellsDbBaselineImpl>(std::move(roots));
+}
+
+//
+// Main implementation
+//
+class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb {
+ public:
+  explicit StaticBagOfCellsDbLazyImpl(std::unique_ptr<BlobView> data, StaticBagOfCellsDbLazy::Options options)
+      : data_(std::move(data)), options_(std::move(options)) {
+    get_thread_safe_counter().add(1);
+  }
+  td::Result<size_t> get_root_count() override {
+    TRY_STATUS(check_status());
+    TRY_STATUS(check_result(load_header()));
+    return info_.root_count;
+  };
+  td::Result<Ref<Cell>> get_root_cell(size_t idx) override {
+    TRY_STATUS(check_status());
+    TRY_RESULT(root_count, get_root_count());
+    if (idx >= root_count) {
+      return td::Status::Error(PSLICE() << "invalid root_cell index: " << idx);
+    }
+    TRY_RESULT(cell_idx, load_root_idx(td::narrow_cast<int>(idx)));
+    // Load DataCell in order to ensure lower hashes correctness
+    // They will be valid for all non-root cell automaically
+    TRY_RESULT(data_cell, check_result(load_data_cell(td::narrow_cast<int>(cell_idx))));
+    return create_root_cell(std::move(data_cell));
+  };
+
+  ~StaticBagOfCellsDbLazyImpl() {
+    //LOG(ERROR) << deserialize_cell_cnt_ << " " << deserialize_cell_hash_cnt_;
+    get_thread_safe_counter().add(-1);
+  }
+
+ private:
+  std::atomic<bool> should_cache_cells_{true};
+  std::unique_ptr<BlobView> data_;
+  StaticBagOfCellsDbLazy::Options options_;
+  bool has_info_{false};
+  BagOfCells::Info info_;
+
+  std::mutex index_i_mutex_;
+  td::RwMutex index_data_rw_mutex_;
+  std::string index_data_;
+  std::atomic<int> index_i_{0};
+  size_t index_offset_{0};
+  DataCellCacheMutex cells_;
+  //DataCellCacheNoop cells_;
+  //DataCellCacheTdlib cells_;
+  int next_idx_{0};
+  Ref<Cell> empty_cell_;
+
+  //stats
+  td::ThreadSafeCounter deserialize_cell_cnt_;
+  td::ThreadSafeCounter deserialize_cell_hash_cnt_;
+
+  std::atomic<bool> has_error_{false};
+  std::mutex status_mutex_;
+  td::Status status_;
+
+  static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() {
+    static auto res = td::NamedThreadSafeCounter::get_default().get_counter("StaticBagOfCellsDbLazy");
+    return res;
+  }
+
+  td::Status check_status() TD_WARN_UNUSED_RESULT {
+    if (has_error_.load(std::memory_order_relaxed)) {
+      std::lock_guard<std::mutex> guard(status_mutex_);
+      return status_.clone();
+    }
+    return td::Status::OK();
+  }
+  template <class T>
+  T check_result(T&& to_check) {
+    CHECK(status_.is_ok());
+    if (to_check.is_error()) {
+      std::lock_guard<std::mutex> guard(status_mutex_);
+      has_error_.store(true);
+      status_ = to_check.error().clone();
+    }
+    return std::forward<T>(to_check);
+  }
+
+  td::Result<Ref<DataCell>> load_by_idx(int idx) override {
+    TRY_STATUS(check_status());
+    return check_result(load_data_cell(idx));
+  }
+
+  struct Ptr {
+    td::MutableSlice as_slice() {
+      return data;
+    }
+    td::string data;
+  };
+  // May be optimized
+  auto alloc(size_t size) {
+    //return td::StackAllocator::alloc(size);
+    return Ptr{std::string(size, '\0')};
+  }
+
+  td::Result<size_t> load_idx_offset(int idx) {
+    if (idx < 0) {
+      return 0;
+    }
+    td::Slice offset_view;
+    CHECK(info_.offset_byte_size <= 8);
+    char arr[8];
+    td::RwMutex::ReadLock guard;
+    if (info_.has_index) {
+      TRY_RESULT(new_offset_view, data_->view(td::MutableSlice(arr, info_.offset_byte_size),
+                                              info_.index_offset + idx * info_.offset_byte_size));
+      offset_view = new_offset_view;
+    } else {
+      guard = index_data_rw_mutex_.lock_read().move_as_ok();
+      offset_view = td::Slice(index_data_).substr(idx * info_.offset_byte_size, info_.offset_byte_size);
+    }
+
+    CHECK(offset_view.size() == (size_t)info_.offset_byte_size);
+    return td::narrow_cast<std::size_t>(info_.read_offset(offset_view.ubegin()));
+  }
+
+  td::Result<td::int64> load_root_idx(int root_i) {
+    CHECK(root_i >= 0 && root_i < info_.root_count);
+    if (!info_.has_roots) {
+      return 0;
+    }
+    char arr[8];
+    TRY_RESULT(idx_view, data_->view(td::MutableSlice(arr, info_.ref_byte_size),
+                                     info_.roots_offset + root_i * info_.ref_byte_size));
+    CHECK(idx_view.size() == (size_t)info_.ref_byte_size);
+    return info_.read_ref(idx_view.ubegin());
+  }
+
+  struct CellLocation {
+    std::size_t begin;
+    std::size_t end;
+    bool should_cache;
+  };
+  td::Result<CellLocation> get_cell_location(int idx) {
+    CHECK(idx >= 0);
+    CHECK(idx < info_.cell_count);
+    TRY_STATUS(preload_index(idx));
+    TRY_RESULT(from, load_idx_offset(idx - 1));
+    TRY_RESULT(till, load_idx_offset(idx));
+    CellLocation res;
+    res.begin = from;
+    res.end = till;
+    res.should_cache = true;
+    if (info_.has_cache_bits) {
+      res.begin /= 2;
+      res.should_cache = res.end % 2 == 1;
+      res.end /= 2;
+    }
+    CHECK(std::numeric_limits<std::size_t>::max() - res.begin >= info_.data_offset);
+    CHECK(std::numeric_limits<std::size_t>::max() - res.end >= info_.data_offset);
+    res.begin += static_cast<std::size_t>(info_.data_offset);
+    res.end += static_cast<std::size_t>(info_.data_offset);
+    return res;
+  }
+
+  td::Status load_header() {
+    if (has_info_) {
+      return td::Status::OK();
+    }
+    std::string header(1000, '\0');
+    TRY_RESULT(header_view, data_->view(td::MutableSlice(header).truncate(data_->size()), 0))
+    auto parse_res = info_.parse_serialized_header(header_view);
+    if (parse_res <= 0) {
+      return td::Status::Error("bag-of-cell error: failed to read header");
+    }
+    if (info_.total_size < data_->size()) {
+      return td::Status::Error("bag-of-cell error: not enough data");
+    }
+    if (options_.check_crc32c && info_.has_crc32c) {
+      std::string buf(td::narrow_cast<std::size_t>(info_.total_size), '\0');
+      TRY_RESULT(data, data_->view(td::MutableSlice(buf), 0));
+      unsigned crc_computed = td::crc32c(td::Slice{data.ubegin(), data.uend() - 4});
+      unsigned crc_stored = td::as<unsigned>(data.uend() - 4);
+      if (crc_computed != crc_stored) {
+        return td::Status::Error(PSLICE()
+                                 << "bag-of-cells CRC32C mismatch: expected " << td::format::as_hex(crc_computed)
+                                 << ", found " << td::format::as_hex(crc_stored));
+      }
+    }
+    has_info_ = true;
+    return td::Status::OK();
+  }
+
+  td::Status preload_index(int idx) {
+    if (info_.has_index) {
+      return td::Status::OK();
+    }
+
+    CHECK(idx < info_.cell_count);
+    if (index_i_.load(std::memory_order_relaxed) > idx) {
+      return td::Status::OK();
+    }
+
+    std::lock_guard<std::mutex> index_i_guard(index_i_mutex_);
+    std::array<char, 1024> buf;
+    auto buf_slice = td::MutableSlice(buf.data(), buf.size());
+    for (; index_i_ <= idx; index_i_++) {
+      auto offset = td::narrow_cast<size_t>(info_.data_offset + index_offset_);
+      CHECK(data_->size() >= offset);
+      TRY_RESULT(cell, data_->view(buf_slice.copy().truncate(data_->size() - offset), offset));
+      CellSerializationInfo cell_info;
+      TRY_STATUS(cell_info.init(cell, info_.ref_byte_size));
+      index_offset_ += cell_info.end_offset;
+      LOG_CHECK((unsigned)info_.offset_byte_size <= 8) << info_.offset_byte_size;
+      td::uint8 tmp[8];
+      info_.write_offset(tmp, index_offset_);
+      auto guard = index_data_rw_mutex_.lock_write();
+      index_data_.append(reinterpret_cast<const char*>(tmp), info_.offset_byte_size);
+    }
+    return td::Status::OK();
+  }
+
+  Ref<Cell> get_any_cell(int idx) {
+    return get_data_cell(idx);
+  }
+
+  Ref<DataCell> get_data_cell(int idx) {
+    return cells_.load(idx);
+  }
+
+  Ref<DataCell> set_data_cell(int idx, Ref<DataCell> cell) {
+    if (/*idx >= info_.root_count || */ !should_cache_cells_.load(std::memory_order_relaxed)) {
+      return cell;
+    }
+    CHECK(cell.not_null());
+    return cells_.store(idx, std::move(cell));
+  }
+
+  Ref<Cell> set_any_cell(int idx, Ref<Cell> cell) {
+    auto data_cell = Ref<DataCell>(cell);
+    if (data_cell.is_null()) {
+      return cell;
+    }
+    return set_data_cell(idx, std::move(data_cell));
+  }
+
+  td::Result<Ref<Cell>> load_any_cell(int idx) {
+    {
+      auto cell = get_any_cell(idx);
+      if (cell.not_null()) {
+        return std::move(cell);
+      }
+    }
+
+    TRY_RESULT(cell_location, get_cell_location(idx));
+    auto buf = alloc(cell_location.end - cell_location.begin);
+    TRY_RESULT(cell_slice, data_->view(buf.as_slice(), cell_location.begin));
+    TRY_RESULT(res, deserialize_any_cell(idx, cell_slice, cell_location.should_cache));
+    return std::move(res);
+  }
+
+  td::Result<Ref<DataCell>> load_data_cell(int idx) {
+    {
+      auto cell = get_data_cell(idx);
+      if (cell.not_null()) {
+        return std::move(cell);
+      }
+    }
+
+    TRY_RESULT(cell_location, get_cell_location(idx));
+    auto buf = alloc(cell_location.end - cell_location.begin);
+    TRY_RESULT(cell_slice, data_->view(buf.as_slice(), cell_location.begin));
+    TRY_RESULT(res, deserialize_data_cell(idx, cell_slice, cell_location.should_cache));
+    return std::move(res);
+  }
+
+  td::Result<Ref<DataCell>> deserialize_data_cell(int idx, td::Slice cell_slice, bool should_cache) {
+    CellSerializationInfo cell_info;
+    TRY_STATUS(cell_info.init(cell_slice, info_.ref_byte_size));
+    if (cell_slice.size() != cell_info.end_offset) {
+      return td::Status::Error(PSLICE() << "unused space in cell #" << idx << " serialization");
+    }
+    return deserialize_data_cell(idx, cell_slice, cell_info, should_cache);
+  }
+
+  td::Result<Ref<DataCell>> deserialize_data_cell(int idx, td::Slice cell_slice, const CellSerializationInfo& cell_info,
+                                                  bool should_cache) {
+    deserialize_cell_cnt_.add(1);
+    Ref<Cell> refs[4];
+    CHECK(cell_info.refs_cnt <= 4);
+    auto* ref_ptr = cell_slice.ubegin() + cell_info.refs_offset;
+    for (int k = 0; k < cell_info.refs_cnt; k++, ref_ptr += info_.ref_byte_size) {
+      int ref_idx = td::narrow_cast<int>(info_.read_ref(ref_ptr));
+      if (ref_idx >= info_.cell_count) {
+        return td::Status::Error(PSLICE() << "invalid bag-of-cells cell #" << idx << " refers to cell #" << ref_idx
+                                          << " which is too big " << td::tag("cell_count", info_.cell_count));
+      }
+      if (idx >= ref_idx) {
+        return td::Status::Error(PSLICE() << "invalid bag-of-cells cell #" << idx << " refers to cell #" << ref_idx
+                                          << " which is a backward reference");
+      }
+      TRY_RESULT(ref, load_any_cell(ref_idx));
+      refs[k] = std::move(ref);
+    }
+
+    TRY_RESULT(data_cell, cell_info.create_data_cell(cell_slice, td::Span<Ref<Cell>>(refs, cell_info.refs_cnt)));
+    if (!should_cache) {
+      return std::move(data_cell);
+    }
+    return set_data_cell(idx, std::move(data_cell));
+  }
+
+  td::Result<Ref<Cell>> deserialize_any_cell(int idx, td::Slice cell_slice, bool should_cache) {
+    CellSerializationInfo cell_info;
+    TRY_STATUS(cell_info.init(cell_slice, info_.ref_byte_size));
+    if (cell_info.with_hashes) {
+      deserialize_cell_hash_cnt_.add(1);
+      int n = cell_info.level_mask.get_hashes_count();
+      return create_ext_cell(cell_info.level_mask, cell_slice.substr(cell_info.hashes_offset, n * Cell::hash_bytes),
+                             cell_slice.substr(cell_info.depth_offset, n * Cell::depth_bytes), idx);
+    }
+    TRY_RESULT(data_cell, deserialize_data_cell(idx, cell_slice, cell_info, should_cache));
+    return std::move(data_cell);
+  }
+  td::Result<Ref<Cell>> create_root_cell(Ref<DataCell> data_cell) {
+    return StaticBocRootCell::create(std::move(data_cell), shared_from_this());
+  }
+};
+
+td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbLazy::create(std::unique_ptr<BlobView> data,
+                                                                               Options options) {
+  return std::make_shared<StaticBagOfCellsDbLazyImpl>(std::move(data), std::move(options));
+}
+
+td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbLazy::create(td::BufferSlice data, Options options) {
+  return std::make_shared<StaticBagOfCellsDbLazyImpl>(vm::BufferSliceBlobView::create(std::move(data)),
+                                                      std::move(options));
+}
+
+td::Result<std::shared_ptr<StaticBagOfCellsDb>> StaticBagOfCellsDbLazy::create(std::string data, Options options) {
+  return create(BufferSliceBlobView::create(td::BufferSlice(data)), std::move(options));
+}
+
+}  // namespace vm

crypto/vm/db/StaticBagOfCellsDb.h

@@ -0,0 +1,60 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "vm/cells.h"
+#include "vm/db/BlobView.h"
+
+#include "td/utils/Status.h"
+
+namespace vm {
+class StaticBagOfCellsDb : public std::enable_shared_from_this<StaticBagOfCellsDb> {
+ public:
+  virtual ~StaticBagOfCellsDb() = default;
+  // TODO: handle errors
+  virtual td::Result<size_t> get_root_count() = 0;
+  virtual td::Result<Ref<Cell>> get_root_cell(size_t idx) = 0;
+
+ protected:
+  virtual td::Result<Ref<DataCell>> load_by_idx(int idx) = 0;
+  friend class StaticBocLoader;
+  friend class StaticBocRootLoader;
+  td::Result<Ref<Cell>> create_ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth, int idx);
+  td::Result<Ref<Cell>> create_root_ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth, int idx);
+};
+
+class StaticBagOfCellsDbBaseline {
+ public:
+  static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(std::unique_ptr<BlobView> data);
+  static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(td::Slice data);
+};
+
+class StaticBagOfCellsDbLazy {
+ public:
+  struct Options {
+    Options() {
+    }
+    bool check_crc32c{false};
+  };
+  static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(std::unique_ptr<BlobView> data, Options options = {});
+  static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(td::BufferSlice data, Options options = {});
+  static td::Result<std::shared_ptr<StaticBagOfCellsDb>> create(std::string data, Options options = {});
+};
+
+}  // namespace vm

crypto/vm/db/TonDb.cpp

@@ -0,0 +1,325 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "vm/db/TonDb.h"
+
+#include "td/utils/tl_helpers.h"
+#include "td/utils/Random.h"
+
+#if TDDB_USE_ROCKSDB
+#include "td/db/RocksDb.h"
+#endif
+
+namespace vm {
+
+template <class StorerT>
+void SmartContractMeta::store(StorerT &storer) const {
+  using td::store;
+  store(stats.cells_total_count, storer);
+  store(stats.cells_total_size, storer);
+  store(type, storer);
+}
+template <class ParserT>
+void SmartContractMeta::parse(ParserT &parser) {
+  using td::parse;
+  parse(stats.cells_total_count, parser);
+  parse(stats.cells_total_size, parser);
+  parse(type, parser);
+}
+
+//
+// SmartContractDbImpl
+//
+Ref<Cell> SmartContractDbImpl::get_root() {
+  if (sync_root_with_db_ || !new_root_.is_null()) {
+    return new_root_;
+  }
+
+  sync_root_with_db();
+  return new_root_;
+}
+
+void SmartContractDbImpl::set_root(Ref<Cell> new_root) {
+  CHECK(new_root.not_null());
+  sync_root_with_db();
+  if (is_dynamic()) {
+    cell_db_->dec(new_root_);
+  }
+  new_root_ = std::move(new_root);
+  if (is_dynamic()) {
+    cell_db_->inc(new_root_);
+  }
+}
+
+SmartContractDbImpl::SmartContractDbImpl(td::Slice hash, std::shared_ptr<KeyValueReader> kv)
+    : hash_(hash.str()), kv_(std::move(kv)) {
+  cell_db_ = DynamicBagOfCellsDb::create();
+}
+
+SmartContractMeta SmartContractDbImpl::get_meta() {
+  sync_root_with_db();
+  return meta_;
+}
+td::Status SmartContractDbImpl::validate_meta() {
+  if (!is_dynamic()) {
+    return td::Status::OK();
+  }
+  sync_root_with_db();
+  TRY_RESULT(in_db, kv_->count({}));
+  if (static_cast<td::int64>(in_db) != meta_.stats.cells_total_count + 2) {
+    return td::Status::Error(PSLICE() << "Invalid meta " << td::tag("expected_count", in_db)
+                                      << td::tag("meta_count", meta_.stats.cells_total_count + 2));
+  }
+  return td::Status::OK();
+}
+
+bool SmartContractDbImpl::is_dynamic() const {
+  return meta_.type == SmartContractMeta::Dynamic;
+}
+
+bool SmartContractDbImpl::is_root_changed() const {
+  return !new_root_.is_null() && (db_root_.is_null() || db_root_->get_hash() != new_root_->get_hash());
+}
+
+void SmartContractDbImpl::sync_root_with_db() {
+  if (sync_root_with_db_) {
+    return;
+  }
+  std::string root_hash;
+  kv_->get("root", root_hash);
+  std::string meta_serialized;
+  kv_->get("meta", meta_serialized);
+  // TODO: proper serialization
+  td::unserialize(meta_, meta_serialized).ignore();
+  sync_root_with_db_ = true;
+
+  if (root_hash.empty()) {
+    meta_.type = SmartContractMeta::Static;
+    //meta_.type = SmartContractMeta::Dynamic;
+  } else {
+    if (is_dynamic()) {
+      //FIXME: error handling
+      db_root_ = cell_db_->load_cell(root_hash).move_as_ok();
+    } else {
+      std::string boc_serialized;
+      kv_->get("boc", boc_serialized);
+      BagOfCells boc;
+      //TODO: check error
+      boc.deserialize(boc_serialized);
+      db_root_ = boc.get_root_cell();
+    }
+    CHECK(db_root_->get_hash().as_slice() == root_hash);
+    new_root_ = db_root_;
+  }
+}
+
+enum { boc_size = 2000 };
+void SmartContractDbImpl::prepare_commit_dynamic(bool force) {
+  if (!is_dynamic()) {
+    CHECK(force);
+    meta_.stats = {};
+    cell_db_->inc(new_root_);
+  }
+  cell_db_->prepare_commit();
+  meta_.stats.apply_diff(cell_db_->get_stats_diff());
+
+  if (!force && meta_.stats.cells_total_size < boc_size) {
+    //LOG(ERROR) << "DYNAMIC -> BOC";
+    return prepare_commit_static(true);
+  }
+  is_dynamic_commit_ = true;
+};
+
+void SmartContractDbImpl::prepare_commit_static(bool force) {
+  BagOfCells boc;
+  boc.add_root(new_root_);
+  boc.import_cells().ensure();  // FIXME
+  if (!force && boc.estimate_serialized_size(15) > boc_size) {
+    //LOG(ERROR) << "BOC -> DYNAMIC ";
+    return prepare_commit_dynamic(true);
+  }
+  if (is_dynamic()) {
+    cell_db_->dec(new_root_);
+    cell_db_->prepare_commit();
+    // stats is invalid now
+  }
+  is_dynamic_commit_ = false;
+  boc_to_commit_ = boc.serialize_to_string(15);
+  meta_.stats = {};
+}
+
+void SmartContractDbImpl::prepare_transaction() {
+  sync_root_with_db();
+  if (!is_root_changed()) {
+    return;
+  }
+
+  if (is_dynamic()) {
+    prepare_commit_dynamic(false);
+  } else {
+    prepare_commit_static(false);
+  }
+}
+
+void SmartContractDbImpl::commit_transaction(KeyValue &kv) {
+  if (!is_root_changed()) {
+    return;
+  }
+
+  if (is_dynamic_commit_) {
+    //LOG(ERROR) << "STORE DYNAMIC";
+    if (!is_dynamic() && db_root_.not_null()) {
+      kv.erase("boc");
+    }
+    CellStorer storer(kv);
+    cell_db_->commit(storer);
+    meta_.type = SmartContractMeta::Dynamic;
+  } else {
+    //LOG(ERROR) << "STORE BOC";
+    if (is_dynamic() && db_root_.not_null()) {
+      //LOG(ERROR) << "Clear Dynamic db";
+      CellStorer storer(kv);
+      cell_db_->commit(storer);
+      cell_db_ = DynamicBagOfCellsDb::create();
+    }
+    meta_.type = SmartContractMeta::Static;
+    kv.set("boc", boc_to_commit_);
+    boc_to_commit_ = {};
+  }
+
+  kv.set("root", new_root_->get_hash().as_slice());
+  kv.set("meta", td::serialize(meta_));
+  db_root_ = new_root_;
+}
+
+void SmartContractDbImpl::set_reader(std::shared_ptr<KeyValueReader> reader) {
+  kv_ = std::move(reader);
+  cell_db_->set_loader(std::make_unique<CellLoader>(kv_));
+}
+
+//
+// TonDbTransactionImpl
+//
+SmartContractDb TonDbTransactionImpl::begin_smartcontract(td::Slice hash) {
+  SmartContractDb res;
+  contracts_.apply(hash, [&](auto &info) {
+    if (!info.is_inited) {
+      info.is_inited = true;
+      info.hash = hash.str();
+      info.smart_contract_db = std::make_unique<SmartContractDbImpl>(hash, nullptr);
+    }
+    LOG_CHECK(info.generation_ != generation_) << "Cannot begin one smartcontract twice during the same transaction";
+    CHECK(info.smart_contract_db);
+    info.smart_contract_db->set_reader(std::make_shared<td::PrefixedKeyValueReader>(reader_, hash));
+    res = std::move(info.smart_contract_db);
+  });
+  return res;
+}
+
+void TonDbTransactionImpl::commit_smartcontract(SmartContractDb txn) {
+  commit_smartcontract(SmartContractDiff(std::move(txn)));
+}
+void TonDbTransactionImpl::commit_smartcontract(SmartContractDiff txn) {
+  {
+    td::PrefixedKeyValue kv(kv_, txn.hash());
+    txn.commit_transaction(kv);
+  }
+  end_smartcontract(txn.extract_smartcontract());
+}
+
+void TonDbTransactionImpl::abort_smartcontract(SmartContractDb txn) {
+  end_smartcontract(std::move(txn));
+}
+void TonDbTransactionImpl::abort_smartcontract(SmartContractDiff txn) {
+  end_smartcontract(txn.extract_smartcontract());
+}
+
+TonDbTransactionImpl::TonDbTransactionImpl(std::shared_ptr<KeyValue> kv) : kv_(std::move(kv)) {
+  CHECK(kv_ != nullptr);
+  reader_.reset(kv_->snapshot().release());
+}
+
+void TonDbTransactionImpl::begin() {
+  kv_->begin_transaction();
+  generation_++;
+}
+void TonDbTransactionImpl::commit() {
+  kv_->commit_transaction();
+  reader_.reset(kv_->snapshot().release());
+}
+void TonDbTransactionImpl::abort() {
+  kv_->abort_transaction();
+}
+void TonDbTransactionImpl::clear_cache() {
+  contracts_ = {};
+}
+
+void TonDbTransactionImpl::end_smartcontract(SmartContractDb smart_contract) {
+  contracts_.apply(smart_contract->hash(), [&](auto &info) {
+    CHECK(info.hash == smart_contract->hash());
+    CHECK(!info.smart_contract_db);
+    info.smart_contract_db = std::move(smart_contract);
+  });
+}
+
+//
+// TonDbImpl
+//
+TonDbImpl::TonDbImpl(std::unique_ptr<KeyValue> kv)
+    : kv_(std::move(kv)), transaction_(std::make_unique<TonDbTransactionImpl>(kv_)) {
+}
+TonDbImpl::~TonDbImpl() {
+  CHECK(transaction_);
+  kv_->flush();
+}
+TonDbTransaction TonDbImpl::begin_transaction() {
+  CHECK(transaction_);
+  transaction_->begin();
+  return std::move(transaction_);
+}
+void TonDbImpl::commit_transaction(TonDbTransaction transaction) {
+  CHECK(!transaction_);
+  CHECK(&transaction->kv() == kv_.get());
+  transaction_ = std::move(transaction);
+  transaction_->commit();
+}
+void TonDbImpl::abort_transaction(TonDbTransaction transaction) {
+  CHECK(!transaction_);
+  CHECK(&transaction->kv() == kv_.get());
+  transaction_ = std::move(transaction);
+  transaction_->abort();
+}
+void TonDbImpl::clear_cache() {
+  CHECK(transaction_);
+  transaction_->clear_cache();
+}
+
+std::string TonDbImpl::stats() const {
+  return kv_->stats();
+}
+
+td::Result<TonDb> TonDbImpl::open(td::Slice path) {
+#if TDDB_USE_ROCKSDB
+  TRY_RESULT(rocksdb, td::RocksDb::open(path.str()));
+  return std::make_unique<TonDbImpl>(std::make_unique<td::RocksDb>(std::move(rocksdb)));
+#else
+  return td::Status::Error("TonDb is not supported in this build");
+#endif
+}
+
+}  // namespace vm

crypto/vm/db/TonDb.h

@@ -0,0 +1,179 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "vm/cellslice.h"
+#include "vm/cells.h"
+#include "vm/boc.h"
+#include "td/db/KeyValue.h"
+#include "vm/db/CellStorage.h"
+#include "vm/db/CellHashTable.h"
+
+#include "td/utils/Slice.h"
+#include "td/utils/Status.h"
+
+namespace vm {
+class SmartContractDbImpl;
+using SmartContractDb = std::unique_ptr<SmartContractDbImpl>;
+using KeyValue = td::KeyValue;
+using KeyValueReader = td::KeyValueReader;
+
+struct SmartContractMeta {
+  DynamicBagOfCellsDb::Stats stats;
+  enum BagOfCellsType { Dynamic, Static } type{Static};
+
+  template <class StorerT>
+  void store(StorerT &storer) const;
+  template <class ParserT>
+  void parse(ParserT &parser);
+};
+
+class SmartContractDbImpl {
+ public:
+  Ref<Cell> get_root();
+  SmartContractMeta get_meta();
+  td::Status validate_meta();
+
+  void set_root(Ref<Cell> new_root);
+
+  SmartContractDbImpl(td::Slice hash, std::shared_ptr<KeyValueReader> kv);
+
+ private:
+  std::string hash_;
+  std::shared_ptr<KeyValueReader> kv_;
+
+  bool sync_root_with_db_{false};
+  Ref<Cell> db_root_;
+  Ref<Cell> new_root_;
+  SmartContractMeta meta_;
+  bool is_dynamic_commit_;
+  std::string boc_to_commit_;
+
+  std::unique_ptr<DynamicBagOfCellsDb> cell_db_;
+  std::unique_ptr<BagOfCells> bag_of_cells_;
+
+  friend class SmartContractDiff;
+  friend class TonDbTransactionImpl;
+
+  void sync_root_with_db();
+
+  td::Slice hash() const {
+    return hash_;
+  }
+
+  void prepare_transaction();
+  void commit_transaction(KeyValue &kv);
+
+  void set_reader(std::shared_ptr<KeyValueReader> reader);
+
+  bool is_dynamic() const;
+  void prepare_commit_dynamic(bool force);
+  void prepare_commit_static(bool force);
+  bool is_root_changed() const;
+};
+
+class SmartContractDiff {
+ public:
+  explicit SmartContractDiff(SmartContractDb db) : db_(std::move(db)) {
+    db_->prepare_transaction();
+  }
+
+  SmartContractDb extract_smartcontract() {
+    return std::move(db_);
+  }
+
+  td::Slice hash() const {
+    return db_->hash();
+  }
+
+  void commit_transaction(KeyValue &kv) {
+    db_->commit_transaction(kv);
+  }
+
+ private:
+  SmartContractDb db_;
+};
+
+class TonDbTransactionImpl;
+using TonDbTransaction = std::unique_ptr<TonDbTransactionImpl>;
+class TonDbTransactionImpl {
+ public:
+  SmartContractDb begin_smartcontract(td::Slice hash = {});
+
+  void commit_smartcontract(SmartContractDb txn);
+  void commit_smartcontract(SmartContractDiff txn);
+
+  void abort_smartcontract(SmartContractDb txn);
+  void abort_smartcontract(SmartContractDiff txn);
+
+  TonDbTransactionImpl(std::shared_ptr<KeyValue> kv);
+
+ private:
+  std::shared_ptr<KeyValue> kv_;
+  std::shared_ptr<KeyValueReader> reader_;
+  td::uint64 generation_{0};
+
+  struct SmartContractInfo {
+    bool is_inited{false};
+    td::uint64 generation_{0};
+    std::string hash;
+    SmartContractDb smart_contract_db;
+    bool operator<(const SmartContractInfo &other) const {
+      return hash < other.hash;
+    }
+    friend bool operator<(const SmartContractInfo &info, td::Slice hash) {
+      return info.hash < hash;
+    }
+    friend bool operator<(td::Slice hash, const SmartContractInfo &info) {
+      return hash < info.hash;
+    }
+  };
+
+  CellHashTable<SmartContractInfo> contracts_;
+
+  KeyValue &kv() {
+    return *kv_;
+  }
+  friend class TonDbImpl;
+
+  void begin();
+  void commit();
+  void abort();
+  void clear_cache();
+
+  void end_smartcontract(SmartContractDb smart_contract);
+};
+
+class TonDbImpl;
+using TonDb = std::unique_ptr<TonDbImpl>;
+class TonDbImpl {
+ public:
+  TonDbImpl(std::unique_ptr<KeyValue> kv);
+  ~TonDbImpl();
+  TonDbTransaction begin_transaction();
+  void commit_transaction(TonDbTransaction transaction);
+  void abort_transaction(TonDbTransaction transaction);
+  void clear_cache();
+  static td::Result<TonDb> open(td::Slice path);
+  std::string stats() const;
+
+ private:
+  std::shared_ptr<KeyValue> kv_;
+  TonDbTransaction transaction_;
+};
+}  // namespace vm