1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00

initial commit

This commit is contained in:
initial commit 2019-09-07 14:03:22 +04:00 committed by vvaltman
commit c2da007f40
1610 changed files with 398047 additions and 0 deletions

View file

@ -0,0 +1,139 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "ChainBuffer.h"
#include "td/utils/buffer.h"
#include "td/db/utils/StreamInterface.h"
namespace td {
namespace detail {
class ChainBuffer : public StreamWriterInterface, public StreamReaderInterface {
public:
using Options = ::td::ChainBuffer::Options;
ChainBuffer(Options options) {
shared_.options_ = options;
reader_.io_slices_.reserve(options.max_io_slices);
reader_.buf_ = writer_.buf_.extract_reader();
}
// StreamReaderInterface
size_t reader_size() override {
reader_.buf_.sync_with_writer();
return reader_.buf_.size();
}
Slice prepare_read() override {
return reader_.buf_.prepare_read();
}
Span<IoSlice> prepare_readv() override {
reader_.io_slices_.clear();
auto it = reader_.buf_.clone();
while (!it.empty() && reader_.io_slices_.size() < reader_.io_slices_.capacity()) {
auto slice = it.prepare_read();
reader_.io_slices_.push_back(as_io_slice(slice));
it.confirm_read(slice.size());
}
return reader_.io_slices_;
}
void confirm_read(size_t size) override {
reader_.buf_.advance(size);
}
void close_reader(Status error) override {
CHECK(!reader_.is_closed_);
reader_.status_ = std::move(error);
reader_.is_closed_.store(true, std::memory_order_release);
}
bool is_writer_closed() const override {
return writer_.is_closed_.load(std::memory_order_acquire);
}
Status &writer_status() override {
CHECK(is_writer_closed());
return writer_.status_;
}
// StreamWriterInterface
size_t writer_size() override {
return writer_.size_;
}
MutableSlice prepare_write() override {
return writer_.buf_.prepare_append(shared_.options_.chunk_size);
}
MutableSlice prepare_write_at_least(size_t size) override {
return writer_.buf_.prepare_append_at_least(size);
}
void confirm_write(size_t size) override {
writer_.buf_.confirm_append(size);
writer_.size_ += size;
}
void append(Slice data) override {
writer_.buf_.append(data, shared_.options_.chunk_size);
writer_.size_ += data.size();
}
void append(BufferSlice data) override {
writer_.size_ += data.size();
writer_.buf_.append(std::move(data));
}
void append(std::string data) override {
append(Slice(data));
}
void close_writer(Status error) override {
CHECK(!writer_.is_closed_);
writer_.status_ = std::move(error);
writer_.is_closed_.store(true, std::memory_order_release);
}
bool is_reader_closed() const override {
return reader_.is_closed_.load(std::memory_order_acquire);
}
Status &reader_status() override {
CHECK(is_reader_closed());
return reader_.status_;
}
private:
struct SharedData {
Options options_;
} shared_;
char pad1[128];
struct ReaderData {
ChainBufferReader buf_;
std::atomic<bool> is_closed_{false};
Status status_;
std::vector<IoSlice> io_slices_;
} reader_;
char pad2[128];
struct WriterData {
ChainBufferWriter buf_;
std::atomic<bool> is_closed_{false};
Status status_;
size_t size_{0};
} writer_;
};
} // namespace detail
std::pair<ChainBuffer::Reader, ChainBuffer::Writer> ChainBuffer::create(Options options) {
auto impl = std::make_shared<detail::ChainBuffer>(options);
return {Reader(impl), Writer(impl)};
}
} // namespace td

View file

@ -0,0 +1,37 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "StreamInterface.h"
namespace td {
class ChainBuffer {
public:
struct Options {
Options() {
}
size_t chunk_size{1024 * 1024 / 8}; // default size of one chunk in chain buffer
size_t max_io_slices{128}; // size of buffer for writev
};
using Reader = StreamReader;
using Writer = StreamWriter;
static std::pair<Reader, Writer> create(Options options = {});
};
} // namespace td

View file

@ -0,0 +1,155 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "CyclicBuffer.h"
#include "td/utils/misc.h"
#include "td/utils/Slice.h"
#include <atomic>
#include <limits>
#include <memory>
namespace td {
namespace detail {
class CyclicBuffer : public StreamWriterInterface, public StreamReaderInterface {
public:
using Options = ::td::CyclicBuffer::Options;
CyclicBuffer(Options options) {
CHECK(options.chunk_size != 0);
CHECK(options.count != 0);
CHECK(options.alignment != 0);
CHECK(options.chunk_size < (std::numeric_limits<size_t>::max() - options.alignment) / options.count);
shared_.options_ = options;
shared_.raw_data_ = std::make_unique<char[]>(options.size() + options.alignment - 1);
auto pos = reinterpret_cast<uint64>(shared_.raw_data_.get());
auto offset = (options.alignment - static_cast<size_t>(pos % options.alignment)) % options.alignment;
CHECK(offset < options.alignment);
shared_.data_ = MutableSlice(shared_.raw_data_.get() + offset, options.size());
}
// StreamReaderInterface
size_t reader_size() override {
auto offset = reader_.pos_.load(std::memory_order_relaxed);
auto size = writer_.pos_.load(std::memory_order_acquire) - offset;
return narrow_cast<size_t>(size);
}
Slice prepare_read() override {
auto offset = reader_.pos_.load(std::memory_order_relaxed);
auto size = narrow_cast<size_t>(writer_.pos_.load(std::memory_order_acquire) - offset);
if (size == 0) {
return {};
}
offset %= (shared_.options_.chunk_size * shared_.options_.count);
return shared_.data_.substr(narrow_cast<size_t>(offset)).truncate(size).truncate(shared_.options_.chunk_size);
}
Span<IoSlice> prepare_readv() override {
reader_.io_slice_ = as_io_slice(prepare_read());
return Span<IoSlice>(&reader_.io_slice_, 1);
}
void confirm_read(size_t size) override {
reader_.pos_.store(reader_.pos_.load(std::memory_order_relaxed) + size);
}
void close_reader(Status error) override {
CHECK(!reader_.is_closed_);
reader_.status_ = std::move(error);
reader_.is_closed_.store(true, std::memory_order_release);
}
bool is_writer_closed() const override {
return writer_.is_closed_.load(std::memory_order_acquire);
}
Status &writer_status() override {
CHECK(is_writer_closed());
return writer_.status_;
}
// StreamWriterInterface
size_t writer_size() override {
auto offset = reader_.pos_.load(std::memory_order_acquire);
auto size = writer_.pos_.load(std::memory_order_relaxed) - offset;
return narrow_cast<size_t>(size);
}
MutableSlice prepare_write() override {
auto max_offset =
reader_.pos_.load(std::memory_order_acquire) + shared_.options_.chunk_size * (shared_.options_.count - 1);
auto offset = writer_.pos_.load(std::memory_order_relaxed);
if (offset > max_offset) {
return {};
}
offset %= (shared_.options_.chunk_size * shared_.options_.count);
return shared_.data_.substr(narrow_cast<size_t>(offset), shared_.options_.chunk_size);
}
MutableSlice prepare_write_at_least(size_t size) override {
UNREACHABLE();
}
void confirm_write(size_t size) override {
writer_.pos_.store(writer_.pos_.load(std::memory_order_relaxed) + size);
}
void append(Slice data) override {
UNREACHABLE();
}
void append(BufferSlice data) override {
UNREACHABLE();
}
void append(std::string data) override {
UNREACHABLE();
}
void close_writer(Status error) override {
CHECK(!writer_.is_closed_);
writer_.status_ = std::move(error);
writer_.is_closed_.store(true, std::memory_order_release);
}
bool is_reader_closed() const override {
return reader_.is_closed_.load(std::memory_order_acquire);
}
Status &reader_status() override {
CHECK(is_reader_closed());
return reader_.status_;
}
private:
struct SharedData {
std::unique_ptr<char[]> raw_data_;
MutableSlice data_;
Options options_;
} shared_;
struct ReaderData {
std::atomic<uint64> pos_{0};
std::atomic<bool> is_closed_{false};
Status status_;
IoSlice io_slice_;
} reader_;
char pad[128];
struct WriterData {
std::atomic<uint64> pos_{0};
std::atomic<bool> is_closed_{false};
Status status_;
} writer_;
};
} // namespace detail
std::pair<CyclicBuffer::Reader, CyclicBuffer::Writer> CyclicBuffer::create(Options options) {
auto impl = std::make_shared<detail::CyclicBuffer>(options);
return {Reader(impl), Writer(impl)};
}
} // namespace td

View file

@ -0,0 +1,48 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "StreamInterface.h"
#include <utility>
namespace td {
class CyclicBuffer {
public:
struct Options {
Options() {
}
size_t chunk_size{1024 * 1024 / 8};
size_t count{16};
size_t alignment{1024};
size_t size() const {
return chunk_size * count;
}
size_t max_writable_size() {
return size() - chunk_size;
}
};
using Reader = StreamReader;
using Writer = StreamWriter;
static std::pair<Reader, Writer> create(Options options = {});
};
} // namespace td

View file

@ -0,0 +1,67 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "FileSyncState.h"
namespace td {
std::pair<FileSyncState::Reader, FileSyncState::Writer> FileSyncState::create() {
auto self = std::make_shared<Self>();
return {Reader(self), Writer(self)};
}
FileSyncState::Reader::Reader(std::shared_ptr<Self> self) : self(std::move(self)) {
}
bool FileSyncState::Reader::set_requested_sync_size(size_t size) const {
if (self->requested_synced_size.load(std::memory_order_relaxed) == size) {
return false;
}
self->requested_synced_size.store(size, std::memory_order_release);
return true;
}
size_t FileSyncState::Reader::synced_size() const {
return self->synced_size;
}
size_t FileSyncState::Reader::flushed_size() const {
return self->flushed_size;
}
FileSyncState::Writer::Writer(std::shared_ptr<Self> self) : self(std::move(self)) {
}
size_t FileSyncState::Writer::get_requested_synced_size() {
return self->requested_synced_size.load(std::memory_order_acquire);
}
bool FileSyncState::Writer::set_synced_size(size_t size) {
if (self->synced_size.load(std::memory_order_relaxed) == size) {
return false;
}
self->synced_size.store(size, std::memory_order_release);
return true;
}
bool FileSyncState::Writer::set_flushed_size(size_t size) {
if (self->flushed_size.load(std::memory_order_relaxed) == size) {
return false;
}
self->flushed_size.store(size, std::memory_order_release);
return true;
}
} // namespace td

View file

@ -0,0 +1,65 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include <utility>
#include <memory>
#include <atomic>
#include "td/utils/common.h"
namespace td {
class FileSyncState {
struct Self;
public:
class Reader {
public:
Reader() = default;
Reader(std::shared_ptr<Self> self);
bool set_requested_sync_size(size_t size) const;
size_t synced_size() const;
size_t flushed_size() const;
private:
std::shared_ptr<Self> self;
};
class Writer {
public:
Writer() = default;
Writer(std::shared_ptr<Self> self);
size_t get_requested_synced_size();
bool set_synced_size(size_t size);
bool set_flushed_size(size_t size);
private:
std::shared_ptr<Self> self;
};
static std::pair<Reader, Writer> create();
private:
struct Self {
std::atomic<size_t> requested_synced_size{0};
std::atomic<size_t> synced_size{0};
std::atomic<size_t> flushed_size{0};
};
};
} // namespace td

View file

@ -0,0 +1,78 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "FileToStreamActor.h"
namespace td {
FileToStreamActor::FileToStreamActor(FileFd fd, StreamWriter writer, Options options)
: fd_(std::move(fd)), writer_(std::move(writer)), options_(options) {
}
void FileToStreamActor::set_callback(td::unique_ptr<Callback> callback) {
callback_ = std::move(callback);
got_more();
}
void FileToStreamActor::got_more() {
if (!callback_) {
return;
}
callback_->got_more();
}
void FileToStreamActor::loop() {
auto dest = writer_.prepare_write();
if (options_.limit != -1) {
if (static_cast<int64>(dest.size()) > options_.limit) {
dest.truncate(narrow_cast<size_t>(options_.limit));
}
}
if (dest.empty()) {
//NB: Owner of CyclicBufer::Reader should notify this actor after each chunk is readed
return;
}
auto r_size = fd_.read(dest);
if (r_size.is_error()) {
writer_.close_writer(r_size.move_as_error());
got_more();
return stop();
}
auto size = r_size.move_as_ok();
writer_.confirm_write(size);
got_more();
if (options_.limit != -1) {
options_.limit -= narrow_cast<int64>(size);
}
if (options_.limit == 0) {
writer_.close_writer(td::Status::OK());
got_more();
return stop();
}
if (size == 0) {
if (options_.read_tail_each < 0) {
writer_.close_writer(td::Status::OK());
got_more();
return stop();
}
alarm_timestamp() = Timestamp::in(options_.read_tail_each);
return;
}
yield();
}
} // namespace td

View file

@ -0,0 +1,53 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "StreamInterface.h"
#include "td/actor/actor.h"
#include "td/utils/port/FileFd.h"
namespace td {
class FileToStreamActor : public td::actor::Actor {
public:
struct Options {
Options() {
}
int64 limit{-1};
double read_tail_each{-1};
};
class Callback {
public:
virtual ~Callback() {
}
virtual void got_more() = 0;
};
FileToStreamActor(FileFd fd, StreamWriter writer, Options options = {});
void set_callback(td::unique_ptr<Callback> callback);
private:
void got_more();
void loop() override;
FileFd fd_;
StreamWriter writer_;
td::unique_ptr<Callback> callback_;
Options options_;
};
} // namespace td

View file

@ -0,0 +1,79 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "StreamInterface.h"
namespace td {
StreamReader::StreamReader(std::shared_ptr<StreamReaderInterface> self) : self(std::move(self)) {
}
size_t StreamReader::reader_size() {
return self->reader_size();
}
Slice StreamReader::prepare_read() {
return self->prepare_read();
}
Span<IoSlice> StreamReader::prepare_readv() {
return self->prepare_readv();
}
void StreamReader::confirm_read(size_t size) {
return self->confirm_read(size);
}
void StreamReader::close_reader(Status error) {
return self->close_reader(std::move(error));
}
bool StreamReader::is_writer_closed() const {
return self->is_writer_closed();
}
Status &StreamReader::writer_status() {
return self->writer_status();
}
StreamWriter::StreamWriter(std::shared_ptr<StreamWriterInterface> self) : self(std::move(self)) {
}
size_t StreamWriter::writer_size() {
return self->writer_size();
}
MutableSlice StreamWriter::prepare_write() {
return self->prepare_write();
}
MutableSlice StreamWriter::prepare_write_at_least(size_t size) {
return self->prepare_write_at_least(size);
}
void StreamWriter::confirm_write(size_t size) {
return self->confirm_write(size);
}
void StreamWriter::append(Slice data) {
return self->append(data);
}
void StreamWriter::append(BufferSlice data) {
return self->append(std::move(data));
}
void StreamWriter::append(std::string data) {
return self->append(std::move(data));
}
void StreamWriter::close_writer(Status error) {
return self->close_writer(std::move(error));
}
bool StreamWriter::is_reader_closed() const {
return self->is_reader_closed();
}
Status &StreamWriter::reader_status() {
return self->reader_status();
}
} // namespace td

View file

@ -0,0 +1,102 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/buffer.h"
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/utils/port/IoSlice.h"
namespace td {
// Generic stream interface
// Will to hide implementations details.
// CyclicBuffer, ChainBuffer, Bounded ChainBuffer, some clever writers. They all should be interchangable
// Most implementaions will assume that reading and writing may happen concurrently
class StreamReaderInterface {
public:
virtual ~StreamReaderInterface() {
}
virtual size_t reader_size() = 0;
virtual Slice prepare_read() = 0;
virtual Span<IoSlice> prepare_readv() = 0;
virtual void confirm_read(size_t size) = 0;
virtual void close_reader(Status error) = 0;
virtual bool is_writer_closed() const = 0;
virtual Status &writer_status() = 0;
};
class StreamWriterInterface {
public:
virtual ~StreamWriterInterface() {
}
virtual size_t writer_size() = 0;
virtual MutableSlice prepare_write() = 0;
virtual MutableSlice prepare_write_at_least(size_t size) = 0;
virtual void confirm_write(size_t size) = 0;
virtual void append(Slice data) = 0;
virtual void append(BufferSlice data) {
append(data.as_slice());
}
virtual void append(std::string data) {
append(Slice(data));
}
virtual void close_writer(Status error) = 0;
virtual bool is_reader_closed() const = 0;
virtual Status &reader_status() = 0;
};
// Hide shared_ptr
class StreamReader : public StreamReaderInterface {
public:
StreamReader() = default;
StreamReader(std::shared_ptr<StreamReaderInterface> self);
size_t reader_size() override;
Slice prepare_read() override;
Span<IoSlice> prepare_readv() override;
void confirm_read(size_t size) override;
void close_reader(Status error) override;
bool is_writer_closed() const override;
Status &writer_status() override;
private:
std::shared_ptr<StreamReaderInterface> self;
};
class StreamWriter : public StreamWriterInterface {
public:
StreamWriter() = default;
StreamWriter(std::shared_ptr<StreamWriterInterface> self);
size_t writer_size() override;
MutableSlice prepare_write() override;
MutableSlice prepare_write_at_least(size_t size) override;
void confirm_write(size_t size) override;
void append(Slice data) override;
void append(BufferSlice data) override;
void append(std::string data) override;
void close_writer(Status error) override;
bool is_reader_closed() const override;
Status &reader_status() override;
private:
std::shared_ptr<StreamWriterInterface> self;
};
} // namespace td

View file

@ -0,0 +1,112 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "StreamToFileActor.h"
namespace td {
StreamToFileActor::StreamToFileActor(StreamReader reader, FileFd fd, FileSyncState::Writer sync_state, Options options)
: reader_(std::move(reader)), fd_(std::move(fd)), sync_state_(std::move(sync_state)) {
}
void StreamToFileActor::set_callback(td::unique_ptr<Callback> callback) {
callback_ = std::move(callback);
callback_->on_sync_state_changed();
}
Result<bool> StreamToFileActor::is_closed() {
if (!reader_.is_writer_closed()) {
return false;
}
return reader_.writer_status().clone();
}
Status StreamToFileActor::do_flush_once() {
auto size = reader_.reader_size();
size_t total_written = 0;
while (total_written < size) {
auto io_slices = reader_.prepare_readv();
TRY_RESULT(written, fd_.writev(io_slices));
reader_.confirm_read(written);
flushed_size_ += written;
total_written += written;
}
return Status::OK();
}
Status StreamToFileActor::do_sync() {
if (flushed_size_ == synced_size_) {
return Status::OK();
}
TRY_STATUS(fd_.sync());
synced_size_ = flushed_size_;
return Status::OK();
}
void StreamToFileActor::schedule_sync() {
if (synced_size_ == flushed_size_) {
return;
}
if (sync_state_.get_requested_synced_size() > synced_size_) {
sync_at_.relax(Timestamp::in(options_.immediate_sync_delay));
} else {
sync_at_.relax(Timestamp::in(options_.lazy_sync_delay));
}
}
Result<bool> StreamToFileActor::do_loop() {
// We must first check if writer is closed and then drain all data from reader
// Otherwise there will be a race and some of data could be lost.
// Also it could be useful to check error and stop immediately.
TRY_RESULT(is_closed, is_closed());
// Flush all data that is awailable on the at the beginning of loop
TRY_STATUS(do_flush_once());
if ((sync_at_ && sync_at_.is_in_past()) || is_closed) {
TRY_STATUS(do_sync());
sync_at_ = {};
}
bool need_update = sync_state_.set_synced_size(synced_size_) | sync_state_.set_flushed_size(flushed_size_);
if (need_update && callback_) {
callback_->on_sync_state_changed();
}
if (reader_.reader_size() == 0 && is_closed) {
return true;
}
schedule_sync();
return false;
}
void StreamToFileActor::start_up() {
schedule_sync();
}
void StreamToFileActor::loop() {
auto r_is_closed = do_loop();
if (r_is_closed.is_error()) {
reader_.close_reader(r_is_closed.move_as_error());
return stop();
} else if (r_is_closed.ok()) {
reader_.close_reader(Status::OK());
return stop();
}
alarm_timestamp() = sync_at_;
}
} // namespace td

View file

@ -0,0 +1,73 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "StreamInterface.h"
#include "FileSyncState.h"
#include "td/utils/Time.h"
#include "td/utils/port/FileFd.h"
#include "td/actor/actor.h"
namespace td {
class StreamToFileActor : public actor::Actor {
public:
struct Options {
Options() {
}
double lazy_sync_delay = 10;
double immediate_sync_delay = 0.001;
};
class Callback {
public:
virtual ~Callback() {
}
virtual void on_sync_state_changed() = 0;
};
StreamToFileActor(StreamReader reader, FileFd fd, FileSyncState::Writer sync_state, Options options = {});
void set_callback(td::unique_ptr<Callback> callback);
private:
StreamReader reader_;
FileFd fd_;
Timestamp sync_at_;
Options options_;
FileSyncState::Writer sync_state_;
unique_ptr<Callback> callback_;
size_t flushed_size_{0};
size_t synced_size_{0};
TD_WARN_UNUSED_RESULT Result<bool> is_closed();
Status do_flush_once();
Status do_sync();
void schedule_sync();
TD_WARN_UNUSED_RESULT Result<bool> do_loop();
void start_up() override;
void loop() override;
};
} // namespace td