initial commit

tddb/td/db/KeyValue.h

@@ -0,0 +1,116 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "td/utils/Status.h"
+#include "td/utils/logging.h"
+namespace td {
+class KeyValueReader {
+ public:
+  virtual ~KeyValueReader() = default;
+  enum class GetStatus : int32 { Ok, NotFound };
+
+  virtual Result<GetStatus> get(Slice key, std::string &value) = 0;
+  virtual Result<size_t> count(Slice prefix) = 0;
+};
+
+class PrefixedKeyValueReader : public KeyValueReader {
+ public:
+  PrefixedKeyValueReader(std::shared_ptr<KeyValueReader> reader, Slice prefix)
+      : reader_(std::move(reader)), prefix_(prefix.str()) {
+  }
+  Result<GetStatus> get(Slice key, std::string &value) override {
+    return reader_->get(PSLICE() << prefix_ << key, value);
+  }
+  Result<size_t> count(Slice prefix) override {
+    return reader_->count(PSLICE() << prefix_ << prefix);
+  }
+
+ private:
+  std::shared_ptr<KeyValueReader> reader_;
+  std::string prefix_;
+};
+
+class KeyValueUtils {
+ public:
+};
+
+class KeyValue : public KeyValueReader {
+ public:
+  virtual Status set(Slice key, Slice value) = 0;
+  virtual Status erase(Slice key) = 0;
+
+  virtual Status begin_transaction() = 0;
+  virtual Status commit_transaction() = 0;
+  virtual Status abort_transaction() = 0;
+  // Desctructor will abort transaction
+
+  virtual std::unique_ptr<KeyValueReader> snapshot() = 0;
+
+  virtual std::string stats() const {
+    return "";
+  }
+  virtual Status flush() {
+    return Status::OK();
+  }
+};
+class PrefixedKeyValue : public KeyValue {
+ public:
+  PrefixedKeyValue(std::shared_ptr<KeyValue> kv, Slice prefix) : kv_(std::move(kv)), prefix_(prefix.str()) {
+  }
+  Result<GetStatus> get(Slice key, std::string &value) override {
+    return kv_->get(PSLICE() << prefix_ << key, value);
+  }
+  Result<size_t> count(Slice prefix) override {
+    return kv_->count(PSLICE() << prefix_ << prefix);
+  }
+  Status set(Slice key, Slice value) override {
+    return kv_->set(PSLICE() << prefix_ << key, value);
+  }
+  Status erase(Slice key) override {
+    return kv_->erase(PSLICE() << prefix_ << key);
+  }
+
+  Status begin_transaction() override {
+    return kv_->begin_transaction();
+  }
+  Status commit_transaction() override {
+    return kv_->commit_transaction();
+  }
+  Status abort_transaction() override {
+    return kv_->abort_transaction();
+  }
+  // Desctructor will abort transaction
+
+  std::unique_ptr<KeyValueReader> snapshot() override {
+    return kv_->snapshot();
+  }
+
+  std::string stats() const override {
+    return kv_->stats();
+  }
+  Status flush() override {
+    return kv_->flush();
+  }
+
+ private:
+  std::shared_ptr<KeyValue> kv_;
+  std::string prefix_;
+};
+
+}  // namespace td

tddb/td/db/KeyValueAsync.h

@@ -0,0 +1,158 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "td/actor/actor.h"
+#include "td/actor/PromiseFuture.h"
+
+#include "td/db/KeyValue.h"
+
+namespace td {
+
+template <class KeyT, class ValueT>
+class KeyValueActor;
+
+template <class KeyT, class ValueT>
+class KeyValueAsync {
+ public:
+  using ActorType = KeyValueActor<KeyT, ValueT>;
+  struct GetResult {
+    KeyValue::GetStatus status;
+    ValueT value;
+  };
+  KeyValueAsync(std::shared_ptr<KeyValue> key_value);
+  void get(KeyT key, Promise<GetResult> promise = {});
+  void set(KeyT key, ValueT value, Promise<Unit> promise = {}, double sync_delay = 0);
+  void erase(KeyT key, Promise<Unit> promise = {}, double sync_delay = 0);
+
+  KeyValueAsync();
+  KeyValueAsync(KeyValueAsync &&);
+  KeyValueAsync &operator=(KeyValueAsync &&);
+  ~KeyValueAsync();
+
+ private:
+  actor::ActorOwn<ActorType> actor_;
+};
+
+template <class KeyT, class ValueT>
+class KeyValueActor : public actor::Actor {
+ public:
+  KeyValueActor(std::shared_ptr<KeyValue> key_value) : key_value_(std::move(key_value)) {
+  }
+
+  void get(KeyT key, Promise<typename KeyValueAsync<KeyT, ValueT>::GetResult> promise) {
+    std::string value;
+    auto r_status = key_value_->get(as_slice(key), value);
+    if (r_status.is_error()) {
+      promise.set_error(r_status.move_as_error());
+      return;
+    }
+    typename KeyValueAsync<KeyT, ValueT>::GetResult result;
+    result.status = r_status.move_as_ok();
+    if (result.status == KeyValue::GetStatus::Ok) {
+      result.value = ValueT(std::move(value));
+    }
+    promise.set_value(std::move(result));
+  }
+  void set(KeyT key, ValueT value, Promise<Unit> promise, double sync_delay) {
+    schedule_sync(std::move(promise), sync_delay);
+    key_value_->set(as_slice(key), as_slice(value));
+  }
+  void erase(KeyT key, Promise<Unit> promise, double sync_delay) {
+    schedule_sync(std::move(promise), sync_delay);
+    key_value_->erase(as_slice(key));
+  }
+
+ private:
+  std::shared_ptr<KeyValue> key_value_;
+  std::vector<Promise<Unit>> pending_promises_;
+  bool need_sync_ = false;
+  bool sync_active_ = false;
+
+  void tear_down() override {
+    sync();
+  }
+  void sync() {
+    if (!need_sync_) {
+      return;
+    }
+    need_sync_ = false;
+    sync_active_ = false;
+    key_value_->commit_transaction();
+    for (auto &promise : pending_promises_) {
+      promise.set_value(Unit());
+    }
+    pending_promises_.clear();
+  }
+  void schedule_sync(Promise<Unit> promise, double sync_delay) {
+    if (!need_sync_) {
+      key_value_->begin_transaction();
+      need_sync_ = true;
+    }
+
+    if (!sync_active_) {
+      if (sync_delay == 0) {
+        send_sync();
+      } else {
+        alarm_timestamp().relax(Timestamp::in(sync_delay));
+      }
+    }
+    if (promise) {
+      pending_promises_.push_back(std::move(promise));
+    }
+  }
+  void alarm() override {
+    if (need_sync_ && !sync_active_) {
+      send_sync();
+    }
+  }
+  void send_sync() {
+    sync_active_ = true;
+    alarm_timestamp() = Timestamp::never();
+    send_closure(actor_id(this), &KeyValueActor<KeyT, ValueT>::sync);
+  }
+};
+
+template <class KeyT, class ValueT>
+KeyValueAsync<KeyT, ValueT>::KeyValueAsync() = default;
+template <class KeyT, class ValueT>
+KeyValueAsync<KeyT, ValueT>::KeyValueAsync(KeyValueAsync &&) = default;
+template <class KeyT, class ValueT>
+KeyValueAsync<KeyT, ValueT> &KeyValueAsync<KeyT, ValueT>::operator=(KeyValueAsync &&) = default;
+template <class KeyT, class ValueT>
+KeyValueAsync<KeyT, ValueT>::~KeyValueAsync() = default;
+
+template <class KeyT, class ValueT>
+KeyValueAsync<KeyT, ValueT>::KeyValueAsync(std::shared_ptr<KeyValue> key_value) {
+  actor_ = actor::create_actor<ActorType>("KeyValueActor", std::move(key_value));
+}
+template <class KeyT, class ValueT>
+void KeyValueAsync<KeyT, ValueT>::get(KeyT key, Promise<GetResult> promise) {
+  send_closure_later(actor_, &ActorType::get, std::move(key), std::move(promise));
+}
+template <class KeyT, class ValueT>
+void KeyValueAsync<KeyT, ValueT>::set(KeyT key, ValueT value, Promise<Unit> promise, double sync_delay) {
+  send_closure_later(actor_, &ActorType::set, std::move(key), std::move(value), std::move(promise), sync_delay);
+}
+template <class KeyT, class ValueT>
+void KeyValueAsync<KeyT, ValueT>::erase(KeyT key, Promise<Unit> promise, double sync_delay) {
+  send_closure_later(actor_, &ActorType::erase, std::move(key), std::move(promise), sync_delay);
+}
+
+}  // namespace td

tddb/td/db/MemoryKeyValue.cpp

@@ -0,0 +1,74 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "td/db/MemoryKeyValue.h"
+
+#include "td/utils/format.h"
+
+namespace td {
+Result<MemoryKeyValue::GetStatus> MemoryKeyValue::get(Slice key, std::string &value) {
+  auto it = map_.find(key);
+  if (it == map_.end()) {
+    return GetStatus::NotFound;
+  }
+  value = it->second;
+  return GetStatus::Ok;
+}
+Status MemoryKeyValue::set(Slice key, Slice value) {
+  map_[key.str()] = value.str();
+  return Status::OK();
+}
+Status MemoryKeyValue::erase(Slice key) {
+  auto it = map_.find(key);
+  if (it != map_.end()) {
+    map_.erase(it);
+  }
+  return Status::OK();
+}
+
+Result<size_t> MemoryKeyValue::count(Slice prefix) {
+  size_t res = 0;
+  for (auto it = map_.lower_bound(prefix); it != map_.end(); it++) {
+    if (Slice(it->first).truncate(prefix.size()) != prefix) {
+      break;
+    }
+    res++;
+  }
+  return res;
+}
+
+std::unique_ptr<KeyValueReader> MemoryKeyValue::snapshot() {
+  auto res = std::make_unique<MemoryKeyValue>();
+  res->map_ = map_;
+  return std::move(res);
+}
+
+std::string MemoryKeyValue::stats() const {
+  return PSTRING() << "MemoryKeyValueStats{" << tag("get_count", get_count_) << "}";
+}
+
+Status MemoryKeyValue::begin_transaction() {
+  UNREACHABLE();
+}
+Status MemoryKeyValue::commit_transaction() {
+  UNREACHABLE();
+}
+Status MemoryKeyValue::abort_transaction() {
+  UNREACHABLE();
+}
+}  // namespace td

tddb/td/db/MemoryKeyValue.h

@@ -0,0 +1,48 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "td/db/KeyValue.h"
+
+#include <map>
+
+namespace td {
+class MemoryKeyValue : public KeyValue {
+ public:
+  Result<GetStatus> get(Slice key, std::string &value) override;
+  Status set(Slice key, Slice value) override;
+  Status erase(Slice key) override;
+  Result<size_t> count(Slice prefix) override;
+
+  Status begin_transaction() override;
+  Status commit_transaction() override;
+  Status abort_transaction() override;
+
+  std::unique_ptr<KeyValueReader> snapshot() override;
+
+  std::string stats() const override;
+
+ private:
+  class Cmp : public std::less<> {
+   public:
+    using is_transparent = void;
+  };
+  std::map<std::string, std::string, Cmp> map_;
+  int64 get_count_{0};
+};
+}  // namespace td

tddb/td/db/RocksDb.cpp

@@ -0,0 +1,199 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "td/db/RocksDb.h"
+
+#include "rocksdb/db.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/write_batch.h"
+#include "rocksdb/utilities/optimistic_transaction_db.h"
+#include "rocksdb/utilities/transaction.h"
+
+namespace td {
+namespace {
+static Status from_rocksdb(rocksdb::Status status) {
+  if (status.ok()) {
+    return Status::OK();
+  }
+  return Status::Error(status.ToString());
+}
+static Slice from_rocksdb(rocksdb::Slice slice) {
+  return Slice(slice.data(), slice.size());
+}
+static rocksdb::Slice to_rocksdb(Slice slice) {
+  return rocksdb::Slice(slice.data(), slice.size());
+}
+}  // namespace
+
+Status RocksDb::destroy(Slice path) {
+  return from_rocksdb(rocksdb::DestroyDB(path.str(), {}));
+}
+
+RocksDb::RocksDb(RocksDb &&) = default;
+RocksDb &RocksDb::operator=(RocksDb &&) = default;
+
+RocksDb::~RocksDb() {
+  if (!db_) {
+    return;
+  }
+  end_snapshot().ensure();
+}
+
+RocksDb RocksDb::clone() const {
+  return RocksDb{db_, statistics_};
+}
+
+Result<RocksDb> RocksDb::open(std::string path) {
+  rocksdb::OptimisticTransactionDB *db;
+  auto statistics = rocksdb::CreateDBStatistics();
+  {
+    rocksdb::Options options;
+    options.manual_wal_flush = true;
+    options.create_if_missing = true;
+    options.max_background_compactions = 4;
+    options.max_background_flushes = 2;
+    options.bytes_per_sync = 1 << 20;
+    options.writable_file_max_buffer_size = 2 << 14;
+    options.statistics = statistics;
+    TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(options, std::move(path), &db)));
+  }
+  return RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB>(db), std::move(statistics));
+}
+
+std::unique_ptr<KeyValueReader> RocksDb::snapshot() {
+  auto res = std::make_unique<RocksDb>(clone());
+  res->begin_snapshot().ensure();
+  return std::move(res);
+}
+
+std::string RocksDb::stats() const {
+  return statistics_->ToString();
+}
+
+Result<RocksDb::GetStatus> RocksDb::get(Slice key, std::string &value) {
+  //LOG(ERROR) << "GET";
+  rocksdb::Status status;
+  if (snapshot_) {
+    rocksdb::ReadOptions options;
+    options.snapshot = snapshot_.get();
+    status = db_->Get(options, to_rocksdb(key), &value);
+  } else if (transaction_) {
+    status = transaction_->Get({}, to_rocksdb(key), &value);
+  } else {
+    status = db_->Get({}, to_rocksdb(key), &value);
+  }
+  if (status.ok()) {
+    return GetStatus::Ok;
+  }
+  if (status.code() == rocksdb::Status::kNotFound) {
+    return GetStatus::NotFound;
+  }
+  return from_rocksdb(status);
+}
+
+Status RocksDb::set(Slice key, Slice value) {
+  if (write_batch_) {
+    return from_rocksdb(write_batch_->Put(to_rocksdb(key), to_rocksdb(value)));
+  }
+  if (transaction_) {
+    return from_rocksdb(transaction_->Put(to_rocksdb(key), to_rocksdb(value)));
+  }
+  return from_rocksdb(db_->Put({}, to_rocksdb(key), to_rocksdb(value)));
+}
+
+Status RocksDb::erase(Slice key) {
+  if (write_batch_) {
+    return from_rocksdb(write_batch_->Delete(to_rocksdb(key)));
+  }
+  if (transaction_) {
+    return from_rocksdb(transaction_->Delete(to_rocksdb(key)));
+  }
+  return from_rocksdb(db_->Delete({}, to_rocksdb(key)));
+}
+
+Result<size_t> RocksDb::count(Slice prefix) {
+  rocksdb::ReadOptions options;
+  options.snapshot = snapshot_.get();
+  std::unique_ptr<rocksdb::Iterator> iterator;
+  if (snapshot_ || !transaction_) {
+    iterator.reset(db_->NewIterator(options));
+  } else {
+    iterator.reset(transaction_->GetIterator(options));
+  }
+
+  size_t res = 0;
+  for (iterator->Seek(to_rocksdb(prefix)); iterator->Valid(); iterator->Next()) {
+    if (from_rocksdb(iterator->key()).truncate(prefix.size()) != prefix) {
+      break;
+    }
+    res++;
+  }
+  if (!iterator->status().ok()) {
+    return from_rocksdb(iterator->status());
+  }
+  return res;
+}
+
+Status RocksDb::begin_transaction() {
+  write_batch_ = std::make_unique<rocksdb::WriteBatch>();
+  //transaction_.reset(db_->BeginTransaction({}, {}));
+  return Status::OK();
+}
+
+Status RocksDb::commit_transaction() {
+  CHECK(write_batch_);
+  auto write_batch = std::move(write_batch_);
+  rocksdb::WriteOptions options;
+  options.sync = true;
+  TRY_STATUS(from_rocksdb(db_->Write(options, write_batch.get())));
+  return Status::OK();
+
+  //CHECK(transaction_);
+  //auto res = from_rocksdb(transaction_->Commit());
+  //transaction_.reset();
+  //return res;
+}
+
+Status RocksDb::abort_transaction() {
+  CHECK(write_batch_);
+  write_batch_.reset();
+  //CHECK(transaction_);
+  //transaction_.reset();
+  return Status::OK();
+}
+
+Status RocksDb::flush() {
+  return from_rocksdb(db_->Flush({}));
+}
+
+Status RocksDb::begin_snapshot() {
+  snapshot_.reset(db_->GetSnapshot());
+  return td::Status::OK();
+}
+
+Status RocksDb::end_snapshot() {
+  if (snapshot_) {
+    db_->ReleaseSnapshot(snapshot_.release());
+  }
+  return td::Status::OK();
+}
+
+RocksDb::RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db, std::shared_ptr<rocksdb::Statistics> statistics)
+    : db_(std::move(db)), statistics_(std::move(statistics)) {
+}
+}  // namespace td

tddb/td/db/RocksDb.h

@@ -0,0 +1,81 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#if !TDDB_USE_ROCKSDB
+#error "RocksDb is not supported"
+#endif
+
+#include "td/db/KeyValue.h"
+#include "td/utils/Status.h"
+
+namespace rocksdb {
+class OptimisticTransactionDB;
+class Transaction;
+class WriteBatch;
+class Snapshot;
+class Statistics;
+}  // namespace rocksdb
+
+namespace td {
+class RocksDb : public KeyValue {
+ public:
+  static Status destroy(Slice path);
+  RocksDb clone() const;
+  static Result<RocksDb> open(std::string path);
+
+  Result<GetStatus> get(Slice key, std::string &value) override;
+  Status set(Slice key, Slice value) override;
+  Status erase(Slice key) override;
+  Result<size_t> count(Slice prefix) override;
+
+  Status begin_transaction() override;
+  Status commit_transaction() override;
+  Status abort_transaction() override;
+  Status flush() override;
+
+  Status begin_snapshot();
+  Status end_snapshot();
+
+  std::unique_ptr<KeyValueReader> snapshot() override;
+  std::string stats() const override;
+
+  RocksDb(RocksDb &&);
+  RocksDb &operator=(RocksDb &&);
+  ~RocksDb();
+
+ private:
+  std::shared_ptr<rocksdb::OptimisticTransactionDB> db_;
+  std::shared_ptr<rocksdb::Statistics> statistics_;
+
+  std::unique_ptr<rocksdb::Transaction> transaction_;
+  std::unique_ptr<rocksdb::WriteBatch> write_batch_;
+  class UnreachableDeleter {
+   public:
+    template <class T>
+    void operator()(T *) {
+      UNREACHABLE();
+    }
+  };
+  std::unique_ptr<const rocksdb::Snapshot, UnreachableDeleter> snapshot_;
+
+  explicit RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db,
+                   std::shared_ptr<rocksdb::Statistics> statistics);
+};
+}  // namespace td

tddb/td/db/binlog/Binlog.cpp

@@ -0,0 +1,349 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "Binlog.h"
+
+#include "BinlogReaderHelper.h"
+
+#include "td/db/utils/StreamInterface.h"
+#include "td/db/utils/ChainBuffer.h"
+#include "td/db/utils/CyclicBuffer.h"
+#include "td/db/utils/FileSyncState.h"
+#include "td/db/utils/StreamToFileActor.h"
+#include "td/db/utils/FileToStreamActor.h"
+
+#include "td/actor/actor.h"
+
+#include "td/utils/misc.h"
+#include "td/utils/port/path.h"
+#include "td/utils/VectorQueue.h"
+
+namespace td {
+namespace {
+class BinlogReplayActor : public actor::Actor {
+ public:
+  BinlogReplayActor(StreamReader stream_reader, actor::ActorOwn<FileToStreamActor> file_to_stream,
+                    std::shared_ptr<BinlogReaderInterface> binlog_reader, Promise<Unit> promise)
+      : stream_reader_(std::move(stream_reader))
+      , file_to_stream_(std::move(file_to_stream))
+      , binlog_reader_(std::move(binlog_reader))
+      , promise_(std::move(promise)) {
+  }
+
+ private:
+  StreamReader stream_reader_;
+  actor::ActorOwn<FileToStreamActor> file_to_stream_;
+  std::shared_ptr<BinlogReaderInterface> binlog_reader_;
+  Promise<Unit> promise_;
+
+  bool is_writer_closed_{false};
+  BinlogReaderHelper binlog_reader_helper_;
+
+  unique_ptr<FileToStreamActor::Callback> create_callback() {
+    class Callback : public FileToStreamActor::Callback {
+     public:
+      Callback(actor::ActorShared<> actor) : actor_(std::move(actor)) {
+      }
+      void got_more() override {
+        send_signals_later(actor_, actor::ActorSignals::wakeup());
+      }
+
+     private:
+      actor::ActorShared<> actor_;
+    };
+    return make_unique<Callback>(actor_shared(this));
+  }
+
+  void start_up() override {
+    send_closure_later(file_to_stream_, &FileToStreamActor::set_callback, create_callback());
+  }
+  void notify_writer() {
+    send_signals_later(file_to_stream_, actor::ActorSignals::wakeup());
+  }
+
+  void loop() override {
+    auto status = do_loop();
+    if (status.is_error()) {
+      stream_reader_.close_reader(status.clone());
+      promise_.set_error(std::move(status));
+      return stop();
+    }
+    if (is_writer_closed_) {
+      stream_reader_.close_reader(Status::OK());
+      promise_.set_value(Unit());
+      return stop();
+    }
+  }
+  Status do_loop() {
+    is_writer_closed_ = stream_reader_.is_writer_closed();
+    if (is_writer_closed_) {
+      TRY_STATUS(std::move(stream_reader_.writer_status()));
+    }
+
+    // TODO: watermark want_more/got_more logic
+    int64 got_size = stream_reader_.reader_size();
+    while (got_size > 0) {
+      auto slice = stream_reader_.prepare_read();
+      TRY_STATUS(binlog_reader_helper_.parse(*binlog_reader_, slice));
+      stream_reader_.confirm_read(slice.size());
+      got_size -= slice.size();
+    }
+    notify_writer();
+
+    if (is_writer_closed_) {
+      if (binlog_reader_helper_.unparsed_size() != 0) {
+        return Status::Error(PSLICE() << "Got " << binlog_reader_helper_.unparsed_size()
+                                      << " unparsed bytes in binlog");
+      }
+    }
+
+    return Status::OK();
+  }
+};
+}  // namespace
+Binlog::Binlog(string path) : path_(std::move(path)) {
+}
+
+Status Binlog::replay_sync(BinlogReaderInterface& binlog_reader) {
+  TRY_RESULT(fd, FileFd::open(path_, FileFd::Flags::Read));
+  // No need to use Cyclic buffer, but CyclicBuffer is important for async version
+  CyclicBuffer::Options options;
+  options.chunk_size = 256;
+  options.count = 1;
+  auto reader_writer = CyclicBuffer::create(options);
+
+  auto buf_reader = std::move(reader_writer.first);
+  auto buf_writer = std::move(reader_writer.second);
+
+  TRY_RESULT(fd_size, fd.get_size());
+
+  BinlogReaderHelper helper;
+  while (fd_size != 0) {
+    auto read_to = buf_writer.prepare_write();
+    if (static_cast<int64>(read_to.size()) > fd_size) {
+      read_to.truncate(narrow_cast<size_t>(fd_size));
+    }
+    TRY_RESULT(read, fd.read(read_to));
+    if (read == 0) {
+      return Status::Error("Unexpected end of file");
+    }
+    fd_size -= read;
+    buf_writer.confirm_write(read);
+
+    auto data = buf_reader.prepare_read();
+    CHECK(data.size() == read);
+    TRY_STATUS(helper.parse(binlog_reader, data));
+    buf_reader.confirm_read(data.size());
+  }
+
+  if (helper.unparsed_size() != 0) {
+    return Status::Error(PSLICE() << "Got " << helper.unparsed_size() << " unparsed bytes in binlog");
+  }
+
+  //TODO: check crc32
+  //TODO: allow binlog truncate
+  return Status::OK();
+}
+
+void Binlog::replay_async(std::shared_ptr<BinlogReaderInterface> binlog_reader, Promise<Unit> promise) {
+  auto r_fd = FileFd::open(path_, FileFd::Flags::Read);
+  if (r_fd.is_error()) {
+    promise.set_error(r_fd.move_as_error());
+    return;
+  }
+  auto fd = r_fd.move_as_ok();
+  CyclicBuffer::Options buf_options;
+  buf_options.chunk_size = 256;
+  auto reader_writer = CyclicBuffer::create(buf_options);
+
+  auto buf_reader = std::move(reader_writer.first);
+  auto buf_writer = std::move(reader_writer.second);
+
+  auto r_fd_size = fd.get_size();
+  if (r_fd_size.is_error()) {
+    promise.set_error(r_fd_size.move_as_error());
+  }
+  auto options = FileToStreamActor::Options{};
+  options.limit = r_fd_size.move_as_ok();
+  auto file_to_stream =
+      actor::create_actor<FileToStreamActor>("FileToStream", std::move(fd), std::move(buf_writer), options);
+  auto stream_to_binlog = actor::create_actor<BinlogReplayActor>(
+      "BinlogReplay", std::move(buf_reader), std::move(file_to_stream), std::move(binlog_reader), std::move(promise));
+  stream_to_binlog.release();
+}
+
+void Binlog::destroy(CSlice path) {
+  td::unlink(path).ignore();
+}
+
+void Binlog::destroy() {
+  destroy(path_);
+}
+
+BinlogWriter::BinlogWriter(std::string path) : path_(std::move(path)) {
+}
+
+Status BinlogWriter::open() {
+  TRY_RESULT(fd, FileFd::open(path_, FileFd::Flags::Write | FileFd::Flags::Append | FileFd::Create));
+  fd_ = std::move(fd);
+  ChainBuffer::Options buf_options;
+  buf_options.max_io_slices = 128;
+  buf_options.chunk_size = 256;
+  auto reader_writer = ChainBuffer::create(buf_options);
+  buf_reader_ = std::move(reader_writer.first);
+  buf_writer_ = std::move(reader_writer.second);
+  return Status::OK();
+}
+
+Status BinlogWriter::lazy_flush() {
+  if (buf_reader_.reader_size() < 512) {
+    return Status::OK();
+  }
+  return flush();
+}
+
+Status BinlogWriter::flush() {
+  while (buf_reader_.reader_size() != 0) {
+    TRY_RESULT(written, fd_.writev(buf_reader_.prepare_readv()));
+    buf_reader_.confirm_read(written);
+  }
+  return Status::OK();
+}
+Status BinlogWriter::sync() {
+  flush();
+  return fd_.sync();
+}
+
+Status BinlogWriter::close() {
+  sync();
+  fd_.close();
+  return Status::OK();
+}
+
+namespace detail {
+class FlushHelperActor : public actor::Actor {
+ public:
+  FlushHelperActor(FileSyncState::Reader sync_state_reader, actor::ActorOwn<StreamToFileActor> actor)
+      : sync_state_reader_(std::move(sync_state_reader)), actor_(std::move(actor)) {
+  }
+  void flush() {
+    //TODO;
+  }
+  void sync(size_t position, Promise<Unit> promise) {
+    sync_state_reader_.set_requested_sync_size(position);
+    if (promise) {
+      queries_.emplace(position, std::move(promise));
+    }
+    send_signals_later(actor_, actor::ActorSignals::wakeup());
+  }
+
+  void close(Promise<> promise) {
+    close_promise_ = std::move(promise);
+    actor_.reset();
+  }
+
+ private:
+  FileSyncState::Reader sync_state_reader_;
+  actor::ActorOwn<StreamToFileActor> actor_;
+  Promise<> close_promise_;
+
+  struct Query {
+    Query(size_t position, Promise<Unit> promise) : position(position), promise(std::move(promise)) {
+    }
+    size_t position;
+    Promise<Unit> promise;
+  };
+  VectorQueue<Query> queries_;
+
+  unique_ptr<StreamToFileActor::Callback> create_callback() {
+    class Callback : public StreamToFileActor::Callback {
+     public:
+      Callback(actor::ActorShared<> actor) : actor_(std::move(actor)) {
+      }
+      void on_sync_state_changed() override {
+        send_signals_later(actor_, actor::ActorSignals::wakeup());
+      }
+
+     private:
+      actor::ActorShared<> actor_;
+    };
+    return make_unique<Callback>(actor_shared(this));
+  }
+
+  void start_up() override {
+    send_closure_later(actor_, &StreamToFileActor::set_callback, create_callback());
+  }
+
+  void loop() override {
+    auto synced_position = sync_state_reader_.synced_size();
+    while (!queries_.empty() && queries_.front().position <= synced_position) {
+      queries_.front().promise.set_value(Unit());
+      queries_.pop();
+    }
+  }
+
+  void hangup_shared() override {
+    stop();
+  }
+  void tear_down() override {
+    if (close_promise_) {
+      close_promise_.set_value(Unit());
+    }
+  }
+};
+}  // namespace detail
+BinlogWriterAsync::BinlogWriterAsync(std::string path) : path_(std::move(path)) {
+}
+BinlogWriterAsync::~BinlogWriterAsync() = default;
+
+Status BinlogWriterAsync::open() {
+  TRY_RESULT(fd, FileFd::open(path_, FileFd::Flags::Write | FileFd::Flags::Append | FileFd::Create));
+  ChainBuffer::Options buf_options;
+  buf_options.max_io_slices = 128;
+  buf_options.chunk_size = 256;
+  auto reader_writer = ChainBuffer::create(buf_options);
+  buf_writer_ = std::move(reader_writer.second);
+
+  auto sync_state_reader_writer = td::FileSyncState::create();
+  auto writer_actor = actor::create_actor<StreamToFileActor>("StreamToFile", std::move(reader_writer.first),
+                                                             std::move(fd), std::move(sync_state_reader_writer.second));
+  writer_actor_ = writer_actor.get();
+  sync_state_reader_ = std::move(sync_state_reader_writer.first);
+
+  flush_helper_actor_ =
+      actor::create_actor<detail::FlushHelperActor>("FlushHelperActor", sync_state_reader_, std::move(writer_actor));
+
+  return Status::OK();
+}
+
+void BinlogWriterAsync::close(Promise<> promise) {
+  send_closure(std::move(flush_helper_actor_), &detail::FlushHelperActor::close, std::move(promise));
+  writer_actor_ = {};
+}
+void BinlogWriterAsync::lazy_flush() {
+  send_signals_later(writer_actor_, actor::ActorSignals::wakeup());
+}
+
+void BinlogWriterAsync::flush() {
+  send_closure(flush_helper_actor_, &detail::FlushHelperActor::flush);
+}
+void BinlogWriterAsync::sync(Promise<Unit> promise) {
+  send_closure(flush_helper_actor_, &detail::FlushHelperActor::sync, buf_writer_.writer_size(), std::move(promise));
+}
+
+}  // namespace td

tddb/td/db/binlog/Binlog.h

@@ -0,0 +1,135 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "BinlogReaderInterface.h"
+
+#include "td/db/utils/FileSyncState.h"
+#include "td/db/utils/StreamInterface.h"
+
+#include "td/actor/actor.h"
+
+#include "td/utils/misc.h"
+#include "td/utils/port/FileFd.h"
+
+namespace td {
+class BinlogReaderInterface;
+class StreamToFileActor;
+namespace detail {
+class FlushHelperActor;
+}  // namespace detail
+class Binlog {
+ public:
+  explicit Binlog(string path);
+
+  Status replay_sync(BinlogReaderInterface& binlog_reader);
+  void replay_async(std::shared_ptr<BinlogReaderInterface> binlog_reader, Promise<Unit> promise);
+
+  static void destroy(CSlice path);
+  void destroy();
+
+ private:
+  string path_;
+};
+
+class BinlogWriter {
+ public:
+  BinlogWriter(std::string path);
+
+  Status open();
+
+  template <class EventT>
+  Status write_event(EventT&& event, BinlogReaderInterface* binlog_reader);
+  Status lazy_flush();
+  Status flush();
+  Status sync();
+
+  Status close();
+
+ private:
+  string path_;
+  FileFd fd_;
+
+  StreamReader buf_reader_;
+  StreamWriter buf_writer_;
+};
+
+class BinlogWriterAsync {
+ public:
+  BinlogWriterAsync(std::string path);
+  ~BinlogWriterAsync();
+
+  Status open();
+
+  template <class EventT>
+  Status write_event(EventT&& event, BinlogReaderInterface* binlog_reader);
+
+  void close(Promise<> promise);
+
+  void lazy_flush();
+
+  void flush();
+  void sync(Promise<Unit> promise = {});
+
+ private:
+  std::string path_;
+  StreamWriter buf_writer_;
+  actor::ActorId<StreamToFileActor> writer_actor_;
+  actor::ActorOwn<detail::FlushHelperActor> flush_helper_actor_;
+
+  FileSyncState::Reader sync_state_reader_;
+};
+
+template <class EventT>
+Status BinlogWriter::write_event(EventT&& event, BinlogReaderInterface* binlog_reader) {
+  int64 need_size = -event.serialize({});
+  auto dest =
+      buf_writer_.prepare_write_at_least(narrow_cast<size_t>(need_size)).truncate(narrow_cast<size_t>(need_size));
+  auto written = event.serialize(dest);
+  CHECK(written == need_size);
+
+  if (binlog_reader != nullptr) {
+    TRY_RESULT(parsed, binlog_reader->parse(dest));
+    binlog_reader->flush();
+    CHECK(parsed == written);
+  }
+
+  buf_writer_.confirm_write(narrow_cast<size_t>(written));
+  return lazy_flush();
+}
+
+template <class EventT>
+Status BinlogWriterAsync::write_event(EventT&& event, BinlogReaderInterface* binlog_reader) {
+  int64 need_size = -event.serialize({});
+  auto dest =
+      buf_writer_.prepare_write_at_least(narrow_cast<size_t>(need_size)).truncate(narrow_cast<size_t>(need_size));
+  auto written = event.serialize(dest);
+  CHECK(written == need_size);
+
+  if (binlog_reader != nullptr) {
+    TRY_RESULT(parsed, binlog_reader->parse(dest));
+    CHECK(parsed == written);
+  }
+
+  buf_writer_.confirm_write(narrow_cast<size_t>(written));
+  lazy_flush();
+  return Status::OK();
+}
+
+}  // namespace td

tddb/td/db/binlog/BinlogReaderHelper.cpp

@@ -0,0 +1,96 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "BinlogReaderHelper.h"
+#include "BinlogReaderInterface.h"
+
+#include "td/utils/misc.h"
+
+namespace td {
+td::Status BinlogReaderHelper::parse(BinlogReaderInterface& reader, td::Slice data) {
+  SCOPE_EXIT {
+    reader.flush();
+  };
+  while (true) {
+    if (expected_prefix_size_ > 0 && expected_prefix_size_ == prefix_size_) {
+      TRY_RESULT(size, reader.parse(MutableSlice(buf_.data(), prefix_size_)));
+      if (size < 0) {
+        if (expected_prefix_size_ > td::narrow_cast<size_t>(-size)) {
+          return td::Status::Error("BinlogReader decreased logevent size estimation (1)");
+        }
+        expected_prefix_size_ = static_cast<size_t>(-size);
+      } else {
+        if (expected_prefix_size_ != td::narrow_cast<size_t>(size)) {
+          return td::Status::Error("BinlogReader changed logevent");
+        }
+        prefix_size_ = 0;
+        expected_prefix_size_ = 0;
+      }
+    }
+
+    if (data.empty()) {
+      break;
+    }
+
+    if (expected_prefix_size_ > 0) {
+      CHECK(expected_prefix_size_ < buf_.size());
+      CHECK(prefix_size_ < expected_prefix_size_);
+      auto got = data.copy().truncate(expected_prefix_size_ - prefix_size_);
+      reader.flush();
+      auto dest = td::MutableSlice(buf_.data(), buf_.size()).substr(prefix_size_);
+      if (dest.size() < got.size()) {
+        return td::Status::Error("Too big logevent");
+      }
+      dest.copy_from(got);
+      prefix_size_ += got.size();
+      data = data.substr(got.size());
+      continue;
+    }
+
+    CHECK(!data.empty());
+
+    TRY_RESULT(size, reader.parse(data));
+    if (size < 0) {
+      expected_prefix_size_ = td::narrow_cast<size_t>(-size);
+      prefix_size_ = data.size();
+      if (expected_prefix_size_ < prefix_size_) {
+        return td::Status::Error("BinlogReader waits for less data than it already has");
+      }
+      if (expected_prefix_size_ > buf_.size()) {
+        return td::Status::Error("BinlogReader waits for too big logevent");
+      }
+      reader.flush();
+      td::MutableSlice(buf_.data(), prefix_size_).copy_from(data);
+      break;
+    }
+    if (size == 0) {
+      return td::Status::Error("BinlogReader parseed nothing and asked for nothing");
+    }
+    if (td::narrow_cast<size_t>(size) > data.size()) {
+      return td::Status::Error("BinlogReader parseed more than was given");
+    }
+    data = data.substr(static_cast<size_t>(size));
+  }
+  return td::Status::OK();
+}
+
+size_t BinlogReaderHelper::unparsed_size() const {
+  return prefix_size_;
+}
+
+}  // namespace td

tddb/td/db/binlog/BinlogReaderHelper.h

@@ -0,0 +1,46 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "td/utils/Status.h"
+#include "td/utils/Slice.h"
+
+namespace td {
+
+class BinlogReaderInterface;
+
+// Usually we have data available in chunks and we can't control chunk's sizes
+// And some events will be in multiple chunks.
+// We suggest that all events are small, chunks are big and only small
+// portion of events will lie on chunk's border.
+// This helper will store this rare events locally and will feed them
+// to BinlogReaderInterface as single memory chunk each.
+class BinlogReaderHelper {
+ public:
+  td::Status parse(BinlogReaderInterface& reader, td::Slice data);
+
+  size_t unparsed_size() const;
+
+ private:
+  alignas(16) std::array<char, 1024> buf_;
+  size_t prefix_size_{0};
+  size_t expected_prefix_size_{0};
+};
+
+}  // namespace td

tddb/td/db/binlog/BinlogReaderInterface.h

@@ -0,0 +1,42 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "td/utils/Slice.h"
+#include "td/utils/Status.h"
+
+namespace td {
+
+class BinlogReaderInterface {
+ public:
+  virtual ~BinlogReaderInterface() {
+  }
+  // returns error or size
+  // negative size means reader expects data.size() to be at least -size
+  // positive size means first size bytes of data are processed and could be skipped
+  virtual td::Result<td::int64> parse(td::Slice data) = 0;
+
+  // called when all passed slices are invalidated
+  // Till it is called reader may resue all slices given to it.
+  // It makes possible to calculate crc32c in larger chunks
+  // TODO: maybe we should just process all data that we can at once
+  virtual void flush() {
+  }
+};
+}  // namespace td

tddb/td/db/utils/ChainBuffer.cpp

@@ -0,0 +1,139 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "ChainBuffer.h"
+
+#include "td/utils/buffer.h"
+#include "td/db/utils/StreamInterface.h"
+
+namespace td {
+namespace detail {
+class ChainBuffer : public StreamWriterInterface, public StreamReaderInterface {
+ public:
+  using Options = ::td::ChainBuffer::Options;
+
+  ChainBuffer(Options options) {
+    shared_.options_ = options;
+    reader_.io_slices_.reserve(options.max_io_slices);
+    reader_.buf_ = writer_.buf_.extract_reader();
+  }
+
+  // StreamReaderInterface
+  size_t reader_size() override {
+    reader_.buf_.sync_with_writer();
+    return reader_.buf_.size();
+  }
+
+  Slice prepare_read() override {
+    return reader_.buf_.prepare_read();
+  }
+  Span<IoSlice> prepare_readv() override {
+    reader_.io_slices_.clear();
+    auto it = reader_.buf_.clone();
+    while (!it.empty() && reader_.io_slices_.size() < reader_.io_slices_.capacity()) {
+      auto slice = it.prepare_read();
+      reader_.io_slices_.push_back(as_io_slice(slice));
+      it.confirm_read(slice.size());
+    }
+    return reader_.io_slices_;
+  }
+  void confirm_read(size_t size) override {
+    reader_.buf_.advance(size);
+  }
+
+  void close_reader(Status error) override {
+    CHECK(!reader_.is_closed_);
+    reader_.status_ = std::move(error);
+    reader_.is_closed_.store(true, std::memory_order_release);
+  }
+  bool is_writer_closed() const override {
+    return writer_.is_closed_.load(std::memory_order_acquire);
+  }
+  Status &writer_status() override {
+    CHECK(is_writer_closed());
+    return writer_.status_;
+  }
+
+  // StreamWriterInterface
+  size_t writer_size() override {
+    return writer_.size_;
+  }
+  MutableSlice prepare_write() override {
+    return writer_.buf_.prepare_append(shared_.options_.chunk_size);
+  }
+  MutableSlice prepare_write_at_least(size_t size) override {
+    return writer_.buf_.prepare_append_at_least(size);
+  }
+  void confirm_write(size_t size) override {
+    writer_.buf_.confirm_append(size);
+    writer_.size_ += size;
+  }
+  void append(Slice data) override {
+    writer_.buf_.append(data, shared_.options_.chunk_size);
+    writer_.size_ += data.size();
+  }
+  void append(BufferSlice data) override {
+    writer_.size_ += data.size();
+    writer_.buf_.append(std::move(data));
+  }
+  void append(std::string data) override {
+    append(Slice(data));
+  }
+  void close_writer(Status error) override {
+    CHECK(!writer_.is_closed_);
+    writer_.status_ = std::move(error);
+    writer_.is_closed_.store(true, std::memory_order_release);
+  }
+  bool is_reader_closed() const override {
+    return reader_.is_closed_.load(std::memory_order_acquire);
+  }
+  Status &reader_status() override {
+    CHECK(is_reader_closed());
+    return reader_.status_;
+  }
+
+ private:
+  struct SharedData {
+    Options options_;
+  } shared_;
+
+  char pad1[128];
+
+  struct ReaderData {
+    ChainBufferReader buf_;
+    std::atomic<bool> is_closed_{false};
+    Status status_;
+    std::vector<IoSlice> io_slices_;
+  } reader_;
+
+  char pad2[128];
+
+  struct WriterData {
+    ChainBufferWriter buf_;
+    std::atomic<bool> is_closed_{false};
+    Status status_;
+    size_t size_{0};
+  } writer_;
+};
+}  // namespace detail
+
+std::pair<ChainBuffer::Reader, ChainBuffer::Writer> ChainBuffer::create(Options options) {
+  auto impl = std::make_shared<detail::ChainBuffer>(options);
+  return {Reader(impl), Writer(impl)};
+}
+}  // namespace td

tddb/td/db/utils/ChainBuffer.h

@@ -0,0 +1,37 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "td/utils/common.h"
+
+#include "StreamInterface.h"
+
+namespace td {
+class ChainBuffer {
+ public:
+  struct Options {
+    Options() {
+    }
+    size_t chunk_size{1024 * 1024 / 8};  // default size of one chunk in chain buffer
+    size_t max_io_slices{128};           // size of buffer for writev
+  };
+  using Reader = StreamReader;
+  using Writer = StreamWriter;
+  static std::pair<Reader, Writer> create(Options options = {});
+};
+}  // namespace td

tddb/td/db/utils/CyclicBuffer.cpp

@@ -0,0 +1,155 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "CyclicBuffer.h"
+
+#include "td/utils/misc.h"
+#include "td/utils/Slice.h"
+
+#include <atomic>
+#include <limits>
+#include <memory>
+
+namespace td {
+namespace detail {
+class CyclicBuffer : public StreamWriterInterface, public StreamReaderInterface {
+ public:
+  using Options = ::td::CyclicBuffer::Options;
+
+  CyclicBuffer(Options options) {
+    CHECK(options.chunk_size != 0);
+    CHECK(options.count != 0);
+    CHECK(options.alignment != 0);
+    CHECK(options.chunk_size < (std::numeric_limits<size_t>::max() - options.alignment) / options.count);
+    shared_.options_ = options;
+    shared_.raw_data_ = std::make_unique<char[]>(options.size() + options.alignment - 1);
+    auto pos = reinterpret_cast<uint64>(shared_.raw_data_.get());
+    auto offset = (options.alignment - static_cast<size_t>(pos % options.alignment)) % options.alignment;
+    CHECK(offset < options.alignment);
+    shared_.data_ = MutableSlice(shared_.raw_data_.get() + offset, options.size());
+  }
+
+  // StreamReaderInterface
+  size_t reader_size() override {
+    auto offset = reader_.pos_.load(std::memory_order_relaxed);
+    auto size = writer_.pos_.load(std::memory_order_acquire) - offset;
+    return narrow_cast<size_t>(size);
+  }
+  Slice prepare_read() override {
+    auto offset = reader_.pos_.load(std::memory_order_relaxed);
+    auto size = narrow_cast<size_t>(writer_.pos_.load(std::memory_order_acquire) - offset);
+    if (size == 0) {
+      return {};
+    }
+    offset %= (shared_.options_.chunk_size * shared_.options_.count);
+    return shared_.data_.substr(narrow_cast<size_t>(offset)).truncate(size).truncate(shared_.options_.chunk_size);
+  }
+  Span<IoSlice> prepare_readv() override {
+    reader_.io_slice_ = as_io_slice(prepare_read());
+    return Span<IoSlice>(&reader_.io_slice_, 1);
+  }
+  void confirm_read(size_t size) override {
+    reader_.pos_.store(reader_.pos_.load(std::memory_order_relaxed) + size);
+  }
+
+  void close_reader(Status error) override {
+    CHECK(!reader_.is_closed_);
+    reader_.status_ = std::move(error);
+    reader_.is_closed_.store(true, std::memory_order_release);
+  }
+  bool is_writer_closed() const override {
+    return writer_.is_closed_.load(std::memory_order_acquire);
+  }
+  Status &writer_status() override {
+    CHECK(is_writer_closed());
+    return writer_.status_;
+  }
+
+  // StreamWriterInterface
+  size_t writer_size() override {
+    auto offset = reader_.pos_.load(std::memory_order_acquire);
+    auto size = writer_.pos_.load(std::memory_order_relaxed) - offset;
+    return narrow_cast<size_t>(size);
+  }
+  MutableSlice prepare_write() override {
+    auto max_offset =
+        reader_.pos_.load(std::memory_order_acquire) + shared_.options_.chunk_size * (shared_.options_.count - 1);
+    auto offset = writer_.pos_.load(std::memory_order_relaxed);
+    if (offset > max_offset) {
+      return {};
+    }
+    offset %= (shared_.options_.chunk_size * shared_.options_.count);
+    return shared_.data_.substr(narrow_cast<size_t>(offset), shared_.options_.chunk_size);
+  }
+  MutableSlice prepare_write_at_least(size_t size) override {
+    UNREACHABLE();
+  }
+  void confirm_write(size_t size) override {
+    writer_.pos_.store(writer_.pos_.load(std::memory_order_relaxed) + size);
+  }
+  void append(Slice data) override {
+    UNREACHABLE();
+  }
+  void append(BufferSlice data) override {
+    UNREACHABLE();
+  }
+  void append(std::string data) override {
+    UNREACHABLE();
+  }
+  void close_writer(Status error) override {
+    CHECK(!writer_.is_closed_);
+    writer_.status_ = std::move(error);
+    writer_.is_closed_.store(true, std::memory_order_release);
+  }
+  bool is_reader_closed() const override {
+    return reader_.is_closed_.load(std::memory_order_acquire);
+  }
+  Status &reader_status() override {
+    CHECK(is_reader_closed());
+    return reader_.status_;
+  }
+
+ private:
+  struct SharedData {
+    std::unique_ptr<char[]> raw_data_;
+    MutableSlice data_;
+    Options options_;
+  } shared_;
+
+  struct ReaderData {
+    std::atomic<uint64> pos_{0};
+    std::atomic<bool> is_closed_{false};
+    Status status_;
+    IoSlice io_slice_;
+  } reader_;
+
+  char pad[128];
+
+  struct WriterData {
+    std::atomic<uint64> pos_{0};
+    std::atomic<bool> is_closed_{false};
+    Status status_;
+  } writer_;
+};
+}  // namespace detail
+
+std::pair<CyclicBuffer::Reader, CyclicBuffer::Writer> CyclicBuffer::create(Options options) {
+  auto impl = std::make_shared<detail::CyclicBuffer>(options);
+  return {Reader(impl), Writer(impl)};
+}
+}  // namespace td

tddb/td/db/utils/CyclicBuffer.h

@@ -0,0 +1,48 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "StreamInterface.h"
+
+#include <utility>
+
+namespace td {
+
+class CyclicBuffer {
+ public:
+  struct Options {
+    Options() {
+    }
+    size_t chunk_size{1024 * 1024 / 8};
+    size_t count{16};
+    size_t alignment{1024};
+
+    size_t size() const {
+      return chunk_size * count;
+    }
+    size_t max_writable_size() {
+      return size() - chunk_size;
+    }
+  };
+  using Reader = StreamReader;
+  using Writer = StreamWriter;
+  static std::pair<Reader, Writer> create(Options options = {});
+};
+
+}  // namespace td

tddb/td/db/utils/FileSyncState.cpp

@@ -0,0 +1,67 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+
+#include "FileSyncState.h"
+namespace td {
+std::pair<FileSyncState::Reader, FileSyncState::Writer> FileSyncState::create() {
+  auto self = std::make_shared<Self>();
+  return {Reader(self), Writer(self)};
+}
+
+FileSyncState::Reader::Reader(std::shared_ptr<Self> self) : self(std::move(self)) {
+}
+bool FileSyncState::Reader::set_requested_sync_size(size_t size) const {
+  if (self->requested_synced_size.load(std::memory_order_relaxed) == size) {
+    return false;
+  }
+  self->requested_synced_size.store(size, std::memory_order_release);
+  return true;
+}
+
+size_t FileSyncState::Reader::synced_size() const {
+  return self->synced_size;
+}
+size_t FileSyncState::Reader::flushed_size() const {
+  return self->flushed_size;
+}
+
+FileSyncState::Writer::Writer(std::shared_ptr<Self> self) : self(std::move(self)) {
+}
+
+size_t FileSyncState::Writer::get_requested_synced_size() {
+  return self->requested_synced_size.load(std::memory_order_acquire);
+}
+
+bool FileSyncState::Writer::set_synced_size(size_t size) {
+  if (self->synced_size.load(std::memory_order_relaxed) == size) {
+    return false;
+  }
+  self->synced_size.store(size, std::memory_order_release);
+  return true;
+}
+
+bool FileSyncState::Writer::set_flushed_size(size_t size) {
+  if (self->flushed_size.load(std::memory_order_relaxed) == size) {
+    return false;
+  }
+  self->flushed_size.store(size, std::memory_order_release);
+  return true;
+}
+
+}  // namespace td

tddb/td/db/utils/FileSyncState.h

@@ -0,0 +1,65 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include <utility>
+#include <memory>
+#include <atomic>
+
+#include "td/utils/common.h"
+
+namespace td {
+class FileSyncState {
+  struct Self;
+
+ public:
+  class Reader {
+   public:
+    Reader() = default;
+    Reader(std::shared_ptr<Self> self);
+    bool set_requested_sync_size(size_t size) const;
+    size_t synced_size() const;
+    size_t flushed_size() const;
+
+   private:
+    std::shared_ptr<Self> self;
+  };
+
+  class Writer {
+   public:
+    Writer() = default;
+    Writer(std::shared_ptr<Self> self);
+    size_t get_requested_synced_size();
+    bool set_synced_size(size_t size);
+    bool set_flushed_size(size_t size);
+
+   private:
+    std::shared_ptr<Self> self;
+  };
+
+  static std::pair<Reader, Writer> create();
+
+ private:
+  struct Self {
+    std::atomic<size_t> requested_synced_size{0};
+
+    std::atomic<size_t> synced_size{0};
+    std::atomic<size_t> flushed_size{0};
+  };
+};
+}  // namespace td

tddb/td/db/utils/FileToStreamActor.cpp

@@ -0,0 +1,78 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "FileToStreamActor.h"
+
+namespace td {
+FileToStreamActor::FileToStreamActor(FileFd fd, StreamWriter writer, Options options)
+    : fd_(std::move(fd)), writer_(std::move(writer)), options_(options) {
+}
+
+void FileToStreamActor::set_callback(td::unique_ptr<Callback> callback) {
+  callback_ = std::move(callback);
+  got_more();
+}
+
+void FileToStreamActor::got_more() {
+  if (!callback_) {
+    return;
+  }
+  callback_->got_more();
+}
+void FileToStreamActor::loop() {
+  auto dest = writer_.prepare_write();
+  if (options_.limit != -1) {
+    if (static_cast<int64>(dest.size()) > options_.limit) {
+      dest.truncate(narrow_cast<size_t>(options_.limit));
+    }
+  }
+  if (dest.empty()) {
+    //NB: Owner of CyclicBufer::Reader should notify this actor after each chunk is readed
+    return;
+  }
+
+  auto r_size = fd_.read(dest);
+  if (r_size.is_error()) {
+    writer_.close_writer(r_size.move_as_error());
+    got_more();
+    return stop();
+  }
+  auto size = r_size.move_as_ok();
+  writer_.confirm_write(size);
+  got_more();
+  if (options_.limit != -1) {
+    options_.limit -= narrow_cast<int64>(size);
+  }
+  if (options_.limit == 0) {
+    writer_.close_writer(td::Status::OK());
+    got_more();
+    return stop();
+  }
+  if (size == 0) {
+    if (options_.read_tail_each < 0) {
+      writer_.close_writer(td::Status::OK());
+      got_more();
+      return stop();
+    }
+    alarm_timestamp() = Timestamp::in(options_.read_tail_each);
+    return;
+  }
+  yield();
+}
+
+}  // namespace td

tddb/td/db/utils/FileToStreamActor.h

@@ -0,0 +1,53 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "StreamInterface.h"
+
+#include "td/actor/actor.h"
+#include "td/utils/port/FileFd.h"
+
+namespace td {
+class FileToStreamActor : public td::actor::Actor {
+ public:
+  struct Options {
+    Options() {
+    }
+    int64 limit{-1};
+    double read_tail_each{-1};
+  };
+  class Callback {
+   public:
+    virtual ~Callback() {
+    }
+    virtual void got_more() = 0;
+  };
+  FileToStreamActor(FileFd fd, StreamWriter writer, Options options = {});
+
+  void set_callback(td::unique_ptr<Callback> callback);
+
+ private:
+  void got_more();
+  void loop() override;
+  FileFd fd_;
+  StreamWriter writer_;
+  td::unique_ptr<Callback> callback_;
+  Options options_;
+};
+}  // namespace td

tddb/td/db/utils/StreamInterface.cpp

@@ -0,0 +1,79 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "StreamInterface.h"
+namespace td {
+StreamReader::StreamReader(std::shared_ptr<StreamReaderInterface> self) : self(std::move(self)) {
+}
+size_t StreamReader::reader_size() {
+  return self->reader_size();
+}
+Slice StreamReader::prepare_read() {
+  return self->prepare_read();
+}
+Span<IoSlice> StreamReader::prepare_readv() {
+  return self->prepare_readv();
+}
+void StreamReader::confirm_read(size_t size) {
+  return self->confirm_read(size);
+}
+
+void StreamReader::close_reader(Status error) {
+  return self->close_reader(std::move(error));
+}
+bool StreamReader::is_writer_closed() const {
+  return self->is_writer_closed();
+}
+Status &StreamReader::writer_status() {
+  return self->writer_status();
+}
+
+StreamWriter::StreamWriter(std::shared_ptr<StreamWriterInterface> self) : self(std::move(self)) {
+}
+size_t StreamWriter::writer_size() {
+  return self->writer_size();
+}
+MutableSlice StreamWriter::prepare_write() {
+  return self->prepare_write();
+}
+MutableSlice StreamWriter::prepare_write_at_least(size_t size) {
+  return self->prepare_write_at_least(size);
+}
+void StreamWriter::confirm_write(size_t size) {
+  return self->confirm_write(size);
+}
+void StreamWriter::append(Slice data) {
+  return self->append(data);
+}
+void StreamWriter::append(BufferSlice data) {
+  return self->append(std::move(data));
+}
+void StreamWriter::append(std::string data) {
+  return self->append(std::move(data));
+}
+
+void StreamWriter::close_writer(Status error) {
+  return self->close_writer(std::move(error));
+}
+bool StreamWriter::is_reader_closed() const {
+  return self->is_reader_closed();
+}
+Status &StreamWriter::reader_status() {
+  return self->reader_status();
+}
+}  // namespace td

tddb/td/db/utils/StreamInterface.h

@@ -0,0 +1,102 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "td/utils/buffer.h"
+#include "td/utils/Slice.h"
+#include "td/utils/Span.h"
+#include "td/utils/port/IoSlice.h"
+
+namespace td {
+// Generic stream interface
+// Will to hide implementations details.
+// CyclicBuffer, ChainBuffer, Bounded ChainBuffer, some clever writers. They all should be interchangable
+// Most implementaions will assume that reading and writing may happen concurrently
+
+class StreamReaderInterface {
+ public:
+  virtual ~StreamReaderInterface() {
+  }
+  virtual size_t reader_size() = 0;
+  virtual Slice prepare_read() = 0;
+  virtual Span<IoSlice> prepare_readv() = 0;
+  virtual void confirm_read(size_t size) = 0;
+
+  virtual void close_reader(Status error) = 0;
+  virtual bool is_writer_closed() const = 0;
+  virtual Status &writer_status() = 0;
+};
+
+class StreamWriterInterface {
+ public:
+  virtual ~StreamWriterInterface() {
+  }
+  virtual size_t writer_size() = 0;
+  virtual MutableSlice prepare_write() = 0;
+  virtual MutableSlice prepare_write_at_least(size_t size) = 0;
+  virtual void confirm_write(size_t size) = 0;
+  virtual void append(Slice data) = 0;
+  virtual void append(BufferSlice data) {
+    append(data.as_slice());
+  }
+  virtual void append(std::string data) {
+    append(Slice(data));
+  }
+
+  virtual void close_writer(Status error) = 0;
+  virtual bool is_reader_closed() const = 0;
+  virtual Status &reader_status() = 0;
+};
+
+// Hide shared_ptr
+class StreamReader : public StreamReaderInterface {
+ public:
+  StreamReader() = default;
+  StreamReader(std::shared_ptr<StreamReaderInterface> self);
+  size_t reader_size() override;
+  Slice prepare_read() override;
+  Span<IoSlice> prepare_readv() override;
+  void confirm_read(size_t size) override;
+  void close_reader(Status error) override;
+  bool is_writer_closed() const override;
+  Status &writer_status() override;
+
+ private:
+  std::shared_ptr<StreamReaderInterface> self;
+};
+
+class StreamWriter : public StreamWriterInterface {
+ public:
+  StreamWriter() = default;
+  StreamWriter(std::shared_ptr<StreamWriterInterface> self);
+  size_t writer_size() override;
+  MutableSlice prepare_write() override;
+  MutableSlice prepare_write_at_least(size_t size) override;
+  void confirm_write(size_t size) override;
+  void append(Slice data) override;
+  void append(BufferSlice data) override;
+  void append(std::string data) override;
+  void close_writer(Status error) override;
+  bool is_reader_closed() const override;
+  Status &reader_status() override;
+
+ private:
+  std::shared_ptr<StreamWriterInterface> self;
+};
+
+}  // namespace td

tddb/td/db/utils/StreamToFileActor.cpp

@@ -0,0 +1,112 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "StreamToFileActor.h"
+
+namespace td {
+StreamToFileActor::StreamToFileActor(StreamReader reader, FileFd fd, FileSyncState::Writer sync_state, Options options)
+    : reader_(std::move(reader)), fd_(std::move(fd)), sync_state_(std::move(sync_state)) {
+}
+void StreamToFileActor::set_callback(td::unique_ptr<Callback> callback) {
+  callback_ = std::move(callback);
+  callback_->on_sync_state_changed();
+}
+
+Result<bool> StreamToFileActor::is_closed() {
+  if (!reader_.is_writer_closed()) {
+    return false;
+  }
+  return reader_.writer_status().clone();
+}
+
+Status StreamToFileActor::do_flush_once() {
+  auto size = reader_.reader_size();
+  size_t total_written = 0;
+  while (total_written < size) {
+    auto io_slices = reader_.prepare_readv();
+    TRY_RESULT(written, fd_.writev(io_slices));
+    reader_.confirm_read(written);
+    flushed_size_ += written;
+    total_written += written;
+  }
+  return Status::OK();
+}
+
+Status StreamToFileActor::do_sync() {
+  if (flushed_size_ == synced_size_) {
+    return Status::OK();
+  }
+  TRY_STATUS(fd_.sync());
+  synced_size_ = flushed_size_;
+  return Status::OK();
+}
+
+void StreamToFileActor::schedule_sync() {
+  if (synced_size_ == flushed_size_) {
+    return;
+  }
+  if (sync_state_.get_requested_synced_size() > synced_size_) {
+    sync_at_.relax(Timestamp::in(options_.immediate_sync_delay));
+  } else {
+    sync_at_.relax(Timestamp::in(options_.lazy_sync_delay));
+  }
+}
+
+Result<bool> StreamToFileActor::do_loop() {
+  // We must first check if writer is closed and then drain all data from reader
+  // Otherwise there will be a race and some of data could be lost.
+  // Also it could be useful to check error and stop immediately.
+  TRY_RESULT(is_closed, is_closed());
+
+  // Flush all data that is awailable on the at the beginning of loop
+  TRY_STATUS(do_flush_once());
+
+  if ((sync_at_ && sync_at_.is_in_past()) || is_closed) {
+    TRY_STATUS(do_sync());
+    sync_at_ = {};
+  }
+
+  bool need_update = sync_state_.set_synced_size(synced_size_) | sync_state_.set_flushed_size(flushed_size_);
+  if (need_update && callback_) {
+    callback_->on_sync_state_changed();
+  }
+
+  if (reader_.reader_size() == 0 && is_closed) {
+    return true;
+  }
+
+  schedule_sync();
+  return false;
+}
+
+void StreamToFileActor::start_up() {
+  schedule_sync();
+}
+
+void StreamToFileActor::loop() {
+  auto r_is_closed = do_loop();
+  if (r_is_closed.is_error()) {
+    reader_.close_reader(r_is_closed.move_as_error());
+    return stop();
+  } else if (r_is_closed.ok()) {
+    reader_.close_reader(Status::OK());
+    return stop();
+  }
+  alarm_timestamp() = sync_at_;
+}
+}  // namespace td

tddb/td/db/utils/StreamToFileActor.h

@@ -0,0 +1,73 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "StreamInterface.h"
+#include "FileSyncState.h"
+
+#include "td/utils/Time.h"
+#include "td/utils/port/FileFd.h"
+
+#include "td/actor/actor.h"
+
+namespace td {
+class StreamToFileActor : public actor::Actor {
+ public:
+  struct Options {
+    Options() {
+    }
+    double lazy_sync_delay = 10;
+    double immediate_sync_delay = 0.001;
+  };
+
+  class Callback {
+   public:
+    virtual ~Callback() {
+    }
+    virtual void on_sync_state_changed() = 0;
+  };
+
+  StreamToFileActor(StreamReader reader, FileFd fd, FileSyncState::Writer sync_state, Options options = {});
+  void set_callback(td::unique_ptr<Callback> callback);
+
+ private:
+  StreamReader reader_;
+  FileFd fd_;
+  Timestamp sync_at_;
+  Options options_;
+  FileSyncState::Writer sync_state_;
+  unique_ptr<Callback> callback_;
+
+  size_t flushed_size_{0};
+  size_t synced_size_{0};
+
+  TD_WARN_UNUSED_RESULT Result<bool> is_closed();
+
+  Status do_flush_once();
+
+  Status do_sync();
+
+  void schedule_sync();
+
+  TD_WARN_UNUSED_RESULT Result<bool> do_loop();
+
+  void start_up() override;
+
+  void loop() override;
+};
+}  // namespace td