initial commit

tddb/td/db/utils/ChainBuffer.cpp

@@ -0,0 +1,139 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "ChainBuffer.h"
+
+#include "td/utils/buffer.h"
+#include "td/db/utils/StreamInterface.h"
+
+namespace td {
+namespace detail {
+class ChainBuffer : public StreamWriterInterface, public StreamReaderInterface {
+ public:
+  using Options = ::td::ChainBuffer::Options;
+
+  ChainBuffer(Options options) {
+    shared_.options_ = options;
+    reader_.io_slices_.reserve(options.max_io_slices);
+    reader_.buf_ = writer_.buf_.extract_reader();
+  }
+
+  // StreamReaderInterface
+  size_t reader_size() override {
+    reader_.buf_.sync_with_writer();
+    return reader_.buf_.size();
+  }
+
+  Slice prepare_read() override {
+    return reader_.buf_.prepare_read();
+  }
+  Span<IoSlice> prepare_readv() override {
+    reader_.io_slices_.clear();
+    auto it = reader_.buf_.clone();
+    while (!it.empty() && reader_.io_slices_.size() < reader_.io_slices_.capacity()) {
+      auto slice = it.prepare_read();
+      reader_.io_slices_.push_back(as_io_slice(slice));
+      it.confirm_read(slice.size());
+    }
+    return reader_.io_slices_;
+  }
+  void confirm_read(size_t size) override {
+    reader_.buf_.advance(size);
+  }
+
+  void close_reader(Status error) override {
+    CHECK(!reader_.is_closed_);
+    reader_.status_ = std::move(error);
+    reader_.is_closed_.store(true, std::memory_order_release);
+  }
+  bool is_writer_closed() const override {
+    return writer_.is_closed_.load(std::memory_order_acquire);
+  }
+  Status &writer_status() override {
+    CHECK(is_writer_closed());
+    return writer_.status_;
+  }
+
+  // StreamWriterInterface
+  size_t writer_size() override {
+    return writer_.size_;
+  }
+  MutableSlice prepare_write() override {
+    return writer_.buf_.prepare_append(shared_.options_.chunk_size);
+  }
+  MutableSlice prepare_write_at_least(size_t size) override {
+    return writer_.buf_.prepare_append_at_least(size);
+  }
+  void confirm_write(size_t size) override {
+    writer_.buf_.confirm_append(size);
+    writer_.size_ += size;
+  }
+  void append(Slice data) override {
+    writer_.buf_.append(data, shared_.options_.chunk_size);
+    writer_.size_ += data.size();
+  }
+  void append(BufferSlice data) override {
+    writer_.size_ += data.size();
+    writer_.buf_.append(std::move(data));
+  }
+  void append(std::string data) override {
+    append(Slice(data));
+  }
+  void close_writer(Status error) override {
+    CHECK(!writer_.is_closed_);
+    writer_.status_ = std::move(error);
+    writer_.is_closed_.store(true, std::memory_order_release);
+  }
+  bool is_reader_closed() const override {
+    return reader_.is_closed_.load(std::memory_order_acquire);
+  }
+  Status &reader_status() override {
+    CHECK(is_reader_closed());
+    return reader_.status_;
+  }
+
+ private:
+  struct SharedData {
+    Options options_;
+  } shared_;
+
+  char pad1[128];
+
+  struct ReaderData {
+    ChainBufferReader buf_;
+    std::atomic<bool> is_closed_{false};
+    Status status_;
+    std::vector<IoSlice> io_slices_;
+  } reader_;
+
+  char pad2[128];
+
+  struct WriterData {
+    ChainBufferWriter buf_;
+    std::atomic<bool> is_closed_{false};
+    Status status_;
+    size_t size_{0};
+  } writer_;
+};
+}  // namespace detail
+
+std::pair<ChainBuffer::Reader, ChainBuffer::Writer> ChainBuffer::create(Options options) {
+  auto impl = std::make_shared<detail::ChainBuffer>(options);
+  return {Reader(impl), Writer(impl)};
+}
+}  // namespace td

tddb/td/db/utils/ChainBuffer.h

@@ -0,0 +1,37 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "td/utils/common.h"
+
+#include "StreamInterface.h"
+
+namespace td {
+class ChainBuffer {
+ public:
+  struct Options {
+    Options() {
+    }
+    size_t chunk_size{1024 * 1024 / 8};  // default size of one chunk in chain buffer
+    size_t max_io_slices{128};           // size of buffer for writev
+  };
+  using Reader = StreamReader;
+  using Writer = StreamWriter;
+  static std::pair<Reader, Writer> create(Options options = {});
+};
+}  // namespace td

tddb/td/db/utils/CyclicBuffer.cpp

@@ -0,0 +1,155 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "CyclicBuffer.h"
+
+#include "td/utils/misc.h"
+#include "td/utils/Slice.h"
+
+#include <atomic>
+#include <limits>
+#include <memory>
+
+namespace td {
+namespace detail {
+class CyclicBuffer : public StreamWriterInterface, public StreamReaderInterface {
+ public:
+  using Options = ::td::CyclicBuffer::Options;
+
+  CyclicBuffer(Options options) {
+    CHECK(options.chunk_size != 0);
+    CHECK(options.count != 0);
+    CHECK(options.alignment != 0);
+    CHECK(options.chunk_size < (std::numeric_limits<size_t>::max() - options.alignment) / options.count);
+    shared_.options_ = options;
+    shared_.raw_data_ = std::make_unique<char[]>(options.size() + options.alignment - 1);
+    auto pos = reinterpret_cast<uint64>(shared_.raw_data_.get());
+    auto offset = (options.alignment - static_cast<size_t>(pos % options.alignment)) % options.alignment;
+    CHECK(offset < options.alignment);
+    shared_.data_ = MutableSlice(shared_.raw_data_.get() + offset, options.size());
+  }
+
+  // StreamReaderInterface
+  size_t reader_size() override {
+    auto offset = reader_.pos_.load(std::memory_order_relaxed);
+    auto size = writer_.pos_.load(std::memory_order_acquire) - offset;
+    return narrow_cast<size_t>(size);
+  }
+  Slice prepare_read() override {
+    auto offset = reader_.pos_.load(std::memory_order_relaxed);
+    auto size = narrow_cast<size_t>(writer_.pos_.load(std::memory_order_acquire) - offset);
+    if (size == 0) {
+      return {};
+    }
+    offset %= (shared_.options_.chunk_size * shared_.options_.count);
+    return shared_.data_.substr(narrow_cast<size_t>(offset)).truncate(size).truncate(shared_.options_.chunk_size);
+  }
+  Span<IoSlice> prepare_readv() override {
+    reader_.io_slice_ = as_io_slice(prepare_read());
+    return Span<IoSlice>(&reader_.io_slice_, 1);
+  }
+  void confirm_read(size_t size) override {
+    reader_.pos_.store(reader_.pos_.load(std::memory_order_relaxed) + size);
+  }
+
+  void close_reader(Status error) override {
+    CHECK(!reader_.is_closed_);
+    reader_.status_ = std::move(error);
+    reader_.is_closed_.store(true, std::memory_order_release);
+  }
+  bool is_writer_closed() const override {
+    return writer_.is_closed_.load(std::memory_order_acquire);
+  }
+  Status &writer_status() override {
+    CHECK(is_writer_closed());
+    return writer_.status_;
+  }
+
+  // StreamWriterInterface
+  size_t writer_size() override {
+    auto offset = reader_.pos_.load(std::memory_order_acquire);
+    auto size = writer_.pos_.load(std::memory_order_relaxed) - offset;
+    return narrow_cast<size_t>(size);
+  }
+  MutableSlice prepare_write() override {
+    auto max_offset =
+        reader_.pos_.load(std::memory_order_acquire) + shared_.options_.chunk_size * (shared_.options_.count - 1);
+    auto offset = writer_.pos_.load(std::memory_order_relaxed);
+    if (offset > max_offset) {
+      return {};
+    }
+    offset %= (shared_.options_.chunk_size * shared_.options_.count);
+    return shared_.data_.substr(narrow_cast<size_t>(offset), shared_.options_.chunk_size);
+  }
+  MutableSlice prepare_write_at_least(size_t size) override {
+    UNREACHABLE();
+  }
+  void confirm_write(size_t size) override {
+    writer_.pos_.store(writer_.pos_.load(std::memory_order_relaxed) + size);
+  }
+  void append(Slice data) override {
+    UNREACHABLE();
+  }
+  void append(BufferSlice data) override {
+    UNREACHABLE();
+  }
+  void append(std::string data) override {
+    UNREACHABLE();
+  }
+  void close_writer(Status error) override {
+    CHECK(!writer_.is_closed_);
+    writer_.status_ = std::move(error);
+    writer_.is_closed_.store(true, std::memory_order_release);
+  }
+  bool is_reader_closed() const override {
+    return reader_.is_closed_.load(std::memory_order_acquire);
+  }
+  Status &reader_status() override {
+    CHECK(is_reader_closed());
+    return reader_.status_;
+  }
+
+ private:
+  struct SharedData {
+    std::unique_ptr<char[]> raw_data_;
+    MutableSlice data_;
+    Options options_;
+  } shared_;
+
+  struct ReaderData {
+    std::atomic<uint64> pos_{0};
+    std::atomic<bool> is_closed_{false};
+    Status status_;
+    IoSlice io_slice_;
+  } reader_;
+
+  char pad[128];
+
+  struct WriterData {
+    std::atomic<uint64> pos_{0};
+    std::atomic<bool> is_closed_{false};
+    Status status_;
+  } writer_;
+};
+}  // namespace detail
+
+std::pair<CyclicBuffer::Reader, CyclicBuffer::Writer> CyclicBuffer::create(Options options) {
+  auto impl = std::make_shared<detail::CyclicBuffer>(options);
+  return {Reader(impl), Writer(impl)};
+}
+}  // namespace td

tddb/td/db/utils/CyclicBuffer.h

@@ -0,0 +1,48 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "StreamInterface.h"
+
+#include <utility>
+
+namespace td {
+
+class CyclicBuffer {
+ public:
+  struct Options {
+    Options() {
+    }
+    size_t chunk_size{1024 * 1024 / 8};
+    size_t count{16};
+    size_t alignment{1024};
+
+    size_t size() const {
+      return chunk_size * count;
+    }
+    size_t max_writable_size() {
+      return size() - chunk_size;
+    }
+  };
+  using Reader = StreamReader;
+  using Writer = StreamWriter;
+  static std::pair<Reader, Writer> create(Options options = {});
+};
+
+}  // namespace td

tddb/td/db/utils/FileSyncState.cpp

@@ -0,0 +1,67 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+
+#include "FileSyncState.h"
+namespace td {
+std::pair<FileSyncState::Reader, FileSyncState::Writer> FileSyncState::create() {
+  auto self = std::make_shared<Self>();
+  return {Reader(self), Writer(self)};
+}
+
+FileSyncState::Reader::Reader(std::shared_ptr<Self> self) : self(std::move(self)) {
+}
+bool FileSyncState::Reader::set_requested_sync_size(size_t size) const {
+  if (self->requested_synced_size.load(std::memory_order_relaxed) == size) {
+    return false;
+  }
+  self->requested_synced_size.store(size, std::memory_order_release);
+  return true;
+}
+
+size_t FileSyncState::Reader::synced_size() const {
+  return self->synced_size;
+}
+size_t FileSyncState::Reader::flushed_size() const {
+  return self->flushed_size;
+}
+
+FileSyncState::Writer::Writer(std::shared_ptr<Self> self) : self(std::move(self)) {
+}
+
+size_t FileSyncState::Writer::get_requested_synced_size() {
+  return self->requested_synced_size.load(std::memory_order_acquire);
+}
+
+bool FileSyncState::Writer::set_synced_size(size_t size) {
+  if (self->synced_size.load(std::memory_order_relaxed) == size) {
+    return false;
+  }
+  self->synced_size.store(size, std::memory_order_release);
+  return true;
+}
+
+bool FileSyncState::Writer::set_flushed_size(size_t size) {
+  if (self->flushed_size.load(std::memory_order_relaxed) == size) {
+    return false;
+  }
+  self->flushed_size.store(size, std::memory_order_release);
+  return true;
+}
+
+}  // namespace td

tddb/td/db/utils/FileSyncState.h

@@ -0,0 +1,65 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include <utility>
+#include <memory>
+#include <atomic>
+
+#include "td/utils/common.h"
+
+namespace td {
+class FileSyncState {
+  struct Self;
+
+ public:
+  class Reader {
+   public:
+    Reader() = default;
+    Reader(std::shared_ptr<Self> self);
+    bool set_requested_sync_size(size_t size) const;
+    size_t synced_size() const;
+    size_t flushed_size() const;
+
+   private:
+    std::shared_ptr<Self> self;
+  };
+
+  class Writer {
+   public:
+    Writer() = default;
+    Writer(std::shared_ptr<Self> self);
+    size_t get_requested_synced_size();
+    bool set_synced_size(size_t size);
+    bool set_flushed_size(size_t size);
+
+   private:
+    std::shared_ptr<Self> self;
+  };
+
+  static std::pair<Reader, Writer> create();
+
+ private:
+  struct Self {
+    std::atomic<size_t> requested_synced_size{0};
+
+    std::atomic<size_t> synced_size{0};
+    std::atomic<size_t> flushed_size{0};
+  };
+};
+}  // namespace td

tddb/td/db/utils/FileToStreamActor.cpp

@@ -0,0 +1,78 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "FileToStreamActor.h"
+
+namespace td {
+FileToStreamActor::FileToStreamActor(FileFd fd, StreamWriter writer, Options options)
+    : fd_(std::move(fd)), writer_(std::move(writer)), options_(options) {
+}
+
+void FileToStreamActor::set_callback(td::unique_ptr<Callback> callback) {
+  callback_ = std::move(callback);
+  got_more();
+}
+
+void FileToStreamActor::got_more() {
+  if (!callback_) {
+    return;
+  }
+  callback_->got_more();
+}
+void FileToStreamActor::loop() {
+  auto dest = writer_.prepare_write();
+  if (options_.limit != -1) {
+    if (static_cast<int64>(dest.size()) > options_.limit) {
+      dest.truncate(narrow_cast<size_t>(options_.limit));
+    }
+  }
+  if (dest.empty()) {
+    //NB: Owner of CyclicBufer::Reader should notify this actor after each chunk is readed
+    return;
+  }
+
+  auto r_size = fd_.read(dest);
+  if (r_size.is_error()) {
+    writer_.close_writer(r_size.move_as_error());
+    got_more();
+    return stop();
+  }
+  auto size = r_size.move_as_ok();
+  writer_.confirm_write(size);
+  got_more();
+  if (options_.limit != -1) {
+    options_.limit -= narrow_cast<int64>(size);
+  }
+  if (options_.limit == 0) {
+    writer_.close_writer(td::Status::OK());
+    got_more();
+    return stop();
+  }
+  if (size == 0) {
+    if (options_.read_tail_each < 0) {
+      writer_.close_writer(td::Status::OK());
+      got_more();
+      return stop();
+    }
+    alarm_timestamp() = Timestamp::in(options_.read_tail_each);
+    return;
+  }
+  yield();
+}
+
+}  // namespace td

tddb/td/db/utils/FileToStreamActor.h

@@ -0,0 +1,53 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+
+#include "StreamInterface.h"
+
+#include "td/actor/actor.h"
+#include "td/utils/port/FileFd.h"
+
+namespace td {
+class FileToStreamActor : public td::actor::Actor {
+ public:
+  struct Options {
+    Options() {
+    }
+    int64 limit{-1};
+    double read_tail_each{-1};
+  };
+  class Callback {
+   public:
+    virtual ~Callback() {
+    }
+    virtual void got_more() = 0;
+  };
+  FileToStreamActor(FileFd fd, StreamWriter writer, Options options = {});
+
+  void set_callback(td::unique_ptr<Callback> callback);
+
+ private:
+  void got_more();
+  void loop() override;
+  FileFd fd_;
+  StreamWriter writer_;
+  td::unique_ptr<Callback> callback_;
+  Options options_;
+};
+}  // namespace td

tddb/td/db/utils/StreamInterface.cpp

@@ -0,0 +1,79 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "StreamInterface.h"
+namespace td {
+StreamReader::StreamReader(std::shared_ptr<StreamReaderInterface> self) : self(std::move(self)) {
+}
+size_t StreamReader::reader_size() {
+  return self->reader_size();
+}
+Slice StreamReader::prepare_read() {
+  return self->prepare_read();
+}
+Span<IoSlice> StreamReader::prepare_readv() {
+  return self->prepare_readv();
+}
+void StreamReader::confirm_read(size_t size) {
+  return self->confirm_read(size);
+}
+
+void StreamReader::close_reader(Status error) {
+  return self->close_reader(std::move(error));
+}
+bool StreamReader::is_writer_closed() const {
+  return self->is_writer_closed();
+}
+Status &StreamReader::writer_status() {
+  return self->writer_status();
+}
+
+StreamWriter::StreamWriter(std::shared_ptr<StreamWriterInterface> self) : self(std::move(self)) {
+}
+size_t StreamWriter::writer_size() {
+  return self->writer_size();
+}
+MutableSlice StreamWriter::prepare_write() {
+  return self->prepare_write();
+}
+MutableSlice StreamWriter::prepare_write_at_least(size_t size) {
+  return self->prepare_write_at_least(size);
+}
+void StreamWriter::confirm_write(size_t size) {
+  return self->confirm_write(size);
+}
+void StreamWriter::append(Slice data) {
+  return self->append(data);
+}
+void StreamWriter::append(BufferSlice data) {
+  return self->append(std::move(data));
+}
+void StreamWriter::append(std::string data) {
+  return self->append(std::move(data));
+}
+
+void StreamWriter::close_writer(Status error) {
+  return self->close_writer(std::move(error));
+}
+bool StreamWriter::is_reader_closed() const {
+  return self->is_reader_closed();
+}
+Status &StreamWriter::reader_status() {
+  return self->reader_status();
+}
+}  // namespace td

tddb/td/db/utils/StreamInterface.h

@@ -0,0 +1,102 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "td/utils/buffer.h"
+#include "td/utils/Slice.h"
+#include "td/utils/Span.h"
+#include "td/utils/port/IoSlice.h"
+
+namespace td {
+// Generic stream interface
+// Will to hide implementations details.
+// CyclicBuffer, ChainBuffer, Bounded ChainBuffer, some clever writers. They all should be interchangable
+// Most implementaions will assume that reading and writing may happen concurrently
+
+class StreamReaderInterface {
+ public:
+  virtual ~StreamReaderInterface() {
+  }
+  virtual size_t reader_size() = 0;
+  virtual Slice prepare_read() = 0;
+  virtual Span<IoSlice> prepare_readv() = 0;
+  virtual void confirm_read(size_t size) = 0;
+
+  virtual void close_reader(Status error) = 0;
+  virtual bool is_writer_closed() const = 0;
+  virtual Status &writer_status() = 0;
+};
+
+class StreamWriterInterface {
+ public:
+  virtual ~StreamWriterInterface() {
+  }
+  virtual size_t writer_size() = 0;
+  virtual MutableSlice prepare_write() = 0;
+  virtual MutableSlice prepare_write_at_least(size_t size) = 0;
+  virtual void confirm_write(size_t size) = 0;
+  virtual void append(Slice data) = 0;
+  virtual void append(BufferSlice data) {
+    append(data.as_slice());
+  }
+  virtual void append(std::string data) {
+    append(Slice(data));
+  }
+
+  virtual void close_writer(Status error) = 0;
+  virtual bool is_reader_closed() const = 0;
+  virtual Status &reader_status() = 0;
+};
+
+// Hide shared_ptr
+class StreamReader : public StreamReaderInterface {
+ public:
+  StreamReader() = default;
+  StreamReader(std::shared_ptr<StreamReaderInterface> self);
+  size_t reader_size() override;
+  Slice prepare_read() override;
+  Span<IoSlice> prepare_readv() override;
+  void confirm_read(size_t size) override;
+  void close_reader(Status error) override;
+  bool is_writer_closed() const override;
+  Status &writer_status() override;
+
+ private:
+  std::shared_ptr<StreamReaderInterface> self;
+};
+
+class StreamWriter : public StreamWriterInterface {
+ public:
+  StreamWriter() = default;
+  StreamWriter(std::shared_ptr<StreamWriterInterface> self);
+  size_t writer_size() override;
+  MutableSlice prepare_write() override;
+  MutableSlice prepare_write_at_least(size_t size) override;
+  void confirm_write(size_t size) override;
+  void append(Slice data) override;
+  void append(BufferSlice data) override;
+  void append(std::string data) override;
+  void close_writer(Status error) override;
+  bool is_reader_closed() const override;
+  Status &reader_status() override;
+
+ private:
+  std::shared_ptr<StreamWriterInterface> self;
+};
+
+}  // namespace td

tddb/td/db/utils/StreamToFileActor.cpp

@@ -0,0 +1,112 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#include "StreamToFileActor.h"
+
+namespace td {
+StreamToFileActor::StreamToFileActor(StreamReader reader, FileFd fd, FileSyncState::Writer sync_state, Options options)
+    : reader_(std::move(reader)), fd_(std::move(fd)), sync_state_(std::move(sync_state)) {
+}
+void StreamToFileActor::set_callback(td::unique_ptr<Callback> callback) {
+  callback_ = std::move(callback);
+  callback_->on_sync_state_changed();
+}
+
+Result<bool> StreamToFileActor::is_closed() {
+  if (!reader_.is_writer_closed()) {
+    return false;
+  }
+  return reader_.writer_status().clone();
+}
+
+Status StreamToFileActor::do_flush_once() {
+  auto size = reader_.reader_size();
+  size_t total_written = 0;
+  while (total_written < size) {
+    auto io_slices = reader_.prepare_readv();
+    TRY_RESULT(written, fd_.writev(io_slices));
+    reader_.confirm_read(written);
+    flushed_size_ += written;
+    total_written += written;
+  }
+  return Status::OK();
+}
+
+Status StreamToFileActor::do_sync() {
+  if (flushed_size_ == synced_size_) {
+    return Status::OK();
+  }
+  TRY_STATUS(fd_.sync());
+  synced_size_ = flushed_size_;
+  return Status::OK();
+}
+
+void StreamToFileActor::schedule_sync() {
+  if (synced_size_ == flushed_size_) {
+    return;
+  }
+  if (sync_state_.get_requested_synced_size() > synced_size_) {
+    sync_at_.relax(Timestamp::in(options_.immediate_sync_delay));
+  } else {
+    sync_at_.relax(Timestamp::in(options_.lazy_sync_delay));
+  }
+}
+
+Result<bool> StreamToFileActor::do_loop() {
+  // We must first check if writer is closed and then drain all data from reader
+  // Otherwise there will be a race and some of data could be lost.
+  // Also it could be useful to check error and stop immediately.
+  TRY_RESULT(is_closed, is_closed());
+
+  // Flush all data that is awailable on the at the beginning of loop
+  TRY_STATUS(do_flush_once());
+
+  if ((sync_at_ && sync_at_.is_in_past()) || is_closed) {
+    TRY_STATUS(do_sync());
+    sync_at_ = {};
+  }
+
+  bool need_update = sync_state_.set_synced_size(synced_size_) | sync_state_.set_flushed_size(flushed_size_);
+  if (need_update && callback_) {
+    callback_->on_sync_state_changed();
+  }
+
+  if (reader_.reader_size() == 0 && is_closed) {
+    return true;
+  }
+
+  schedule_sync();
+  return false;
+}
+
+void StreamToFileActor::start_up() {
+  schedule_sync();
+}
+
+void StreamToFileActor::loop() {
+  auto r_is_closed = do_loop();
+  if (r_is_closed.is_error()) {
+    reader_.close_reader(r_is_closed.move_as_error());
+    return stop();
+  } else if (r_is_closed.ok()) {
+    reader_.close_reader(Status::OK());
+    return stop();
+  }
+  alarm_timestamp() = sync_at_;
+}
+}  // namespace td

tddb/td/db/utils/StreamToFileActor.h

@@ -0,0 +1,73 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2017-2019 Telegram Systems LLP
+*/
+#pragma once
+#include "StreamInterface.h"
+#include "FileSyncState.h"
+
+#include "td/utils/Time.h"
+#include "td/utils/port/FileFd.h"
+
+#include "td/actor/actor.h"
+
+namespace td {
+class StreamToFileActor : public actor::Actor {
+ public:
+  struct Options {
+    Options() {
+    }
+    double lazy_sync_delay = 10;
+    double immediate_sync_delay = 0.001;
+  };
+
+  class Callback {
+   public:
+    virtual ~Callback() {
+    }
+    virtual void on_sync_state_changed() = 0;
+  };
+
+  StreamToFileActor(StreamReader reader, FileFd fd, FileSyncState::Writer sync_state, Options options = {});
+  void set_callback(td::unique_ptr<Callback> callback);
+
+ private:
+  StreamReader reader_;
+  FileFd fd_;
+  Timestamp sync_at_;
+  Options options_;
+  FileSyncState::Writer sync_state_;
+  unique_ptr<Callback> callback_;
+
+  size_t flushed_size_{0};
+  size_t synced_size_{0};
+
+  TD_WARN_UNUSED_RESULT Result<bool> is_closed();
+
+  Status do_flush_once();
+
+  Status do_sync();
+
+  void schedule_sync();
+
+  TD_WARN_UNUSED_RESULT Result<bool> do_loop();
+
+  void start_up() override;
+
+  void loop() override;
+};
+}  // namespace td