mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
initial commit
This commit is contained in:
commit
c2da007f40
1610 changed files with 398047 additions and 0 deletions
139
tddb/td/db/utils/ChainBuffer.cpp
Normal file
139
tddb/td/db/utils/ChainBuffer.cpp
Normal file
|
@ -0,0 +1,139 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
#include "ChainBuffer.h"
|
||||
|
||||
#include "td/utils/buffer.h"
|
||||
#include "td/db/utils/StreamInterface.h"
|
||||
|
||||
namespace td {
|
||||
namespace detail {
|
||||
class ChainBuffer : public StreamWriterInterface, public StreamReaderInterface {
|
||||
public:
|
||||
using Options = ::td::ChainBuffer::Options;
|
||||
|
||||
ChainBuffer(Options options) {
|
||||
shared_.options_ = options;
|
||||
reader_.io_slices_.reserve(options.max_io_slices);
|
||||
reader_.buf_ = writer_.buf_.extract_reader();
|
||||
}
|
||||
|
||||
// StreamReaderInterface
|
||||
size_t reader_size() override {
|
||||
reader_.buf_.sync_with_writer();
|
||||
return reader_.buf_.size();
|
||||
}
|
||||
|
||||
Slice prepare_read() override {
|
||||
return reader_.buf_.prepare_read();
|
||||
}
|
||||
Span<IoSlice> prepare_readv() override {
|
||||
reader_.io_slices_.clear();
|
||||
auto it = reader_.buf_.clone();
|
||||
while (!it.empty() && reader_.io_slices_.size() < reader_.io_slices_.capacity()) {
|
||||
auto slice = it.prepare_read();
|
||||
reader_.io_slices_.push_back(as_io_slice(slice));
|
||||
it.confirm_read(slice.size());
|
||||
}
|
||||
return reader_.io_slices_;
|
||||
}
|
||||
void confirm_read(size_t size) override {
|
||||
reader_.buf_.advance(size);
|
||||
}
|
||||
|
||||
void close_reader(Status error) override {
|
||||
CHECK(!reader_.is_closed_);
|
||||
reader_.status_ = std::move(error);
|
||||
reader_.is_closed_.store(true, std::memory_order_release);
|
||||
}
|
||||
bool is_writer_closed() const override {
|
||||
return writer_.is_closed_.load(std::memory_order_acquire);
|
||||
}
|
||||
Status &writer_status() override {
|
||||
CHECK(is_writer_closed());
|
||||
return writer_.status_;
|
||||
}
|
||||
|
||||
// StreamWriterInterface
|
||||
size_t writer_size() override {
|
||||
return writer_.size_;
|
||||
}
|
||||
MutableSlice prepare_write() override {
|
||||
return writer_.buf_.prepare_append(shared_.options_.chunk_size);
|
||||
}
|
||||
MutableSlice prepare_write_at_least(size_t size) override {
|
||||
return writer_.buf_.prepare_append_at_least(size);
|
||||
}
|
||||
void confirm_write(size_t size) override {
|
||||
writer_.buf_.confirm_append(size);
|
||||
writer_.size_ += size;
|
||||
}
|
||||
void append(Slice data) override {
|
||||
writer_.buf_.append(data, shared_.options_.chunk_size);
|
||||
writer_.size_ += data.size();
|
||||
}
|
||||
void append(BufferSlice data) override {
|
||||
writer_.size_ += data.size();
|
||||
writer_.buf_.append(std::move(data));
|
||||
}
|
||||
void append(std::string data) override {
|
||||
append(Slice(data));
|
||||
}
|
||||
void close_writer(Status error) override {
|
||||
CHECK(!writer_.is_closed_);
|
||||
writer_.status_ = std::move(error);
|
||||
writer_.is_closed_.store(true, std::memory_order_release);
|
||||
}
|
||||
bool is_reader_closed() const override {
|
||||
return reader_.is_closed_.load(std::memory_order_acquire);
|
||||
}
|
||||
Status &reader_status() override {
|
||||
CHECK(is_reader_closed());
|
||||
return reader_.status_;
|
||||
}
|
||||
|
||||
private:
|
||||
struct SharedData {
|
||||
Options options_;
|
||||
} shared_;
|
||||
|
||||
char pad1[128];
|
||||
|
||||
struct ReaderData {
|
||||
ChainBufferReader buf_;
|
||||
std::atomic<bool> is_closed_{false};
|
||||
Status status_;
|
||||
std::vector<IoSlice> io_slices_;
|
||||
} reader_;
|
||||
|
||||
char pad2[128];
|
||||
|
||||
struct WriterData {
|
||||
ChainBufferWriter buf_;
|
||||
std::atomic<bool> is_closed_{false};
|
||||
Status status_;
|
||||
size_t size_{0};
|
||||
} writer_;
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
std::pair<ChainBuffer::Reader, ChainBuffer::Writer> ChainBuffer::create(Options options) {
|
||||
auto impl = std::make_shared<detail::ChainBuffer>(options);
|
||||
return {Reader(impl), Writer(impl)};
|
||||
}
|
||||
} // namespace td
|
37
tddb/td/db/utils/ChainBuffer.h
Normal file
37
tddb/td/db/utils/ChainBuffer.h
Normal file
|
@ -0,0 +1,37 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
#pragma once
|
||||
#include "td/utils/common.h"
|
||||
|
||||
#include "StreamInterface.h"
|
||||
|
||||
namespace td {
|
||||
class ChainBuffer {
|
||||
public:
|
||||
struct Options {
|
||||
Options() {
|
||||
}
|
||||
size_t chunk_size{1024 * 1024 / 8}; // default size of one chunk in chain buffer
|
||||
size_t max_io_slices{128}; // size of buffer for writev
|
||||
};
|
||||
using Reader = StreamReader;
|
||||
using Writer = StreamWriter;
|
||||
static std::pair<Reader, Writer> create(Options options = {});
|
||||
};
|
||||
} // namespace td
|
155
tddb/td/db/utils/CyclicBuffer.cpp
Normal file
155
tddb/td/db/utils/CyclicBuffer.cpp
Normal file
|
@ -0,0 +1,155 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
#include "CyclicBuffer.h"
|
||||
|
||||
#include "td/utils/misc.h"
|
||||
#include "td/utils/Slice.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
|
||||
namespace td {
|
||||
namespace detail {
|
||||
class CyclicBuffer : public StreamWriterInterface, public StreamReaderInterface {
|
||||
public:
|
||||
using Options = ::td::CyclicBuffer::Options;
|
||||
|
||||
CyclicBuffer(Options options) {
|
||||
CHECK(options.chunk_size != 0);
|
||||
CHECK(options.count != 0);
|
||||
CHECK(options.alignment != 0);
|
||||
CHECK(options.chunk_size < (std::numeric_limits<size_t>::max() - options.alignment) / options.count);
|
||||
shared_.options_ = options;
|
||||
shared_.raw_data_ = std::make_unique<char[]>(options.size() + options.alignment - 1);
|
||||
auto pos = reinterpret_cast<uint64>(shared_.raw_data_.get());
|
||||
auto offset = (options.alignment - static_cast<size_t>(pos % options.alignment)) % options.alignment;
|
||||
CHECK(offset < options.alignment);
|
||||
shared_.data_ = MutableSlice(shared_.raw_data_.get() + offset, options.size());
|
||||
}
|
||||
|
||||
// StreamReaderInterface
|
||||
size_t reader_size() override {
|
||||
auto offset = reader_.pos_.load(std::memory_order_relaxed);
|
||||
auto size = writer_.pos_.load(std::memory_order_acquire) - offset;
|
||||
return narrow_cast<size_t>(size);
|
||||
}
|
||||
Slice prepare_read() override {
|
||||
auto offset = reader_.pos_.load(std::memory_order_relaxed);
|
||||
auto size = narrow_cast<size_t>(writer_.pos_.load(std::memory_order_acquire) - offset);
|
||||
if (size == 0) {
|
||||
return {};
|
||||
}
|
||||
offset %= (shared_.options_.chunk_size * shared_.options_.count);
|
||||
return shared_.data_.substr(narrow_cast<size_t>(offset)).truncate(size).truncate(shared_.options_.chunk_size);
|
||||
}
|
||||
Span<IoSlice> prepare_readv() override {
|
||||
reader_.io_slice_ = as_io_slice(prepare_read());
|
||||
return Span<IoSlice>(&reader_.io_slice_, 1);
|
||||
}
|
||||
void confirm_read(size_t size) override {
|
||||
reader_.pos_.store(reader_.pos_.load(std::memory_order_relaxed) + size);
|
||||
}
|
||||
|
||||
void close_reader(Status error) override {
|
||||
CHECK(!reader_.is_closed_);
|
||||
reader_.status_ = std::move(error);
|
||||
reader_.is_closed_.store(true, std::memory_order_release);
|
||||
}
|
||||
bool is_writer_closed() const override {
|
||||
return writer_.is_closed_.load(std::memory_order_acquire);
|
||||
}
|
||||
Status &writer_status() override {
|
||||
CHECK(is_writer_closed());
|
||||
return writer_.status_;
|
||||
}
|
||||
|
||||
// StreamWriterInterface
|
||||
size_t writer_size() override {
|
||||
auto offset = reader_.pos_.load(std::memory_order_acquire);
|
||||
auto size = writer_.pos_.load(std::memory_order_relaxed) - offset;
|
||||
return narrow_cast<size_t>(size);
|
||||
}
|
||||
MutableSlice prepare_write() override {
|
||||
auto max_offset =
|
||||
reader_.pos_.load(std::memory_order_acquire) + shared_.options_.chunk_size * (shared_.options_.count - 1);
|
||||
auto offset = writer_.pos_.load(std::memory_order_relaxed);
|
||||
if (offset > max_offset) {
|
||||
return {};
|
||||
}
|
||||
offset %= (shared_.options_.chunk_size * shared_.options_.count);
|
||||
return shared_.data_.substr(narrow_cast<size_t>(offset), shared_.options_.chunk_size);
|
||||
}
|
||||
MutableSlice prepare_write_at_least(size_t size) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void confirm_write(size_t size) override {
|
||||
writer_.pos_.store(writer_.pos_.load(std::memory_order_relaxed) + size);
|
||||
}
|
||||
void append(Slice data) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void append(BufferSlice data) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void append(std::string data) override {
|
||||
UNREACHABLE();
|
||||
}
|
||||
void close_writer(Status error) override {
|
||||
CHECK(!writer_.is_closed_);
|
||||
writer_.status_ = std::move(error);
|
||||
writer_.is_closed_.store(true, std::memory_order_release);
|
||||
}
|
||||
bool is_reader_closed() const override {
|
||||
return reader_.is_closed_.load(std::memory_order_acquire);
|
||||
}
|
||||
Status &reader_status() override {
|
||||
CHECK(is_reader_closed());
|
||||
return reader_.status_;
|
||||
}
|
||||
|
||||
private:
|
||||
struct SharedData {
|
||||
std::unique_ptr<char[]> raw_data_;
|
||||
MutableSlice data_;
|
||||
Options options_;
|
||||
} shared_;
|
||||
|
||||
struct ReaderData {
|
||||
std::atomic<uint64> pos_{0};
|
||||
std::atomic<bool> is_closed_{false};
|
||||
Status status_;
|
||||
IoSlice io_slice_;
|
||||
} reader_;
|
||||
|
||||
char pad[128];
|
||||
|
||||
struct WriterData {
|
||||
std::atomic<uint64> pos_{0};
|
||||
std::atomic<bool> is_closed_{false};
|
||||
Status status_;
|
||||
} writer_;
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
std::pair<CyclicBuffer::Reader, CyclicBuffer::Writer> CyclicBuffer::create(Options options) {
|
||||
auto impl = std::make_shared<detail::CyclicBuffer>(options);
|
||||
return {Reader(impl), Writer(impl)};
|
||||
}
|
||||
} // namespace td
|
48
tddb/td/db/utils/CyclicBuffer.h
Normal file
48
tddb/td/db/utils/CyclicBuffer.h
Normal file
|
@ -0,0 +1,48 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "StreamInterface.h"
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace td {
|
||||
|
||||
class CyclicBuffer {
|
||||
public:
|
||||
struct Options {
|
||||
Options() {
|
||||
}
|
||||
size_t chunk_size{1024 * 1024 / 8};
|
||||
size_t count{16};
|
||||
size_t alignment{1024};
|
||||
|
||||
size_t size() const {
|
||||
return chunk_size * count;
|
||||
}
|
||||
size_t max_writable_size() {
|
||||
return size() - chunk_size;
|
||||
}
|
||||
};
|
||||
using Reader = StreamReader;
|
||||
using Writer = StreamWriter;
|
||||
static std::pair<Reader, Writer> create(Options options = {});
|
||||
};
|
||||
|
||||
} // namespace td
|
67
tddb/td/db/utils/FileSyncState.cpp
Normal file
67
tddb/td/db/utils/FileSyncState.cpp
Normal file
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
|
||||
#include "FileSyncState.h"
|
||||
namespace td {
|
||||
std::pair<FileSyncState::Reader, FileSyncState::Writer> FileSyncState::create() {
|
||||
auto self = std::make_shared<Self>();
|
||||
return {Reader(self), Writer(self)};
|
||||
}
|
||||
|
||||
FileSyncState::Reader::Reader(std::shared_ptr<Self> self) : self(std::move(self)) {
|
||||
}
|
||||
bool FileSyncState::Reader::set_requested_sync_size(size_t size) const {
|
||||
if (self->requested_synced_size.load(std::memory_order_relaxed) == size) {
|
||||
return false;
|
||||
}
|
||||
self->requested_synced_size.store(size, std::memory_order_release);
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t FileSyncState::Reader::synced_size() const {
|
||||
return self->synced_size;
|
||||
}
|
||||
size_t FileSyncState::Reader::flushed_size() const {
|
||||
return self->flushed_size;
|
||||
}
|
||||
|
||||
FileSyncState::Writer::Writer(std::shared_ptr<Self> self) : self(std::move(self)) {
|
||||
}
|
||||
|
||||
size_t FileSyncState::Writer::get_requested_synced_size() {
|
||||
return self->requested_synced_size.load(std::memory_order_acquire);
|
||||
}
|
||||
|
||||
bool FileSyncState::Writer::set_synced_size(size_t size) {
|
||||
if (self->synced_size.load(std::memory_order_relaxed) == size) {
|
||||
return false;
|
||||
}
|
||||
self->synced_size.store(size, std::memory_order_release);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool FileSyncState::Writer::set_flushed_size(size_t size) {
|
||||
if (self->flushed_size.load(std::memory_order_relaxed) == size) {
|
||||
return false;
|
||||
}
|
||||
self->flushed_size.store(size, std::memory_order_release);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace td
|
65
tddb/td/db/utils/FileSyncState.h
Normal file
65
tddb/td/db/utils/FileSyncState.h
Normal file
|
@ -0,0 +1,65 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
#pragma once
|
||||
#include <utility>
|
||||
#include <memory>
|
||||
#include <atomic>
|
||||
|
||||
#include "td/utils/common.h"
|
||||
|
||||
namespace td {
|
||||
class FileSyncState {
|
||||
struct Self;
|
||||
|
||||
public:
|
||||
class Reader {
|
||||
public:
|
||||
Reader() = default;
|
||||
Reader(std::shared_ptr<Self> self);
|
||||
bool set_requested_sync_size(size_t size) const;
|
||||
size_t synced_size() const;
|
||||
size_t flushed_size() const;
|
||||
|
||||
private:
|
||||
std::shared_ptr<Self> self;
|
||||
};
|
||||
|
||||
class Writer {
|
||||
public:
|
||||
Writer() = default;
|
||||
Writer(std::shared_ptr<Self> self);
|
||||
size_t get_requested_synced_size();
|
||||
bool set_synced_size(size_t size);
|
||||
bool set_flushed_size(size_t size);
|
||||
|
||||
private:
|
||||
std::shared_ptr<Self> self;
|
||||
};
|
||||
|
||||
static std::pair<Reader, Writer> create();
|
||||
|
||||
private:
|
||||
struct Self {
|
||||
std::atomic<size_t> requested_synced_size{0};
|
||||
|
||||
std::atomic<size_t> synced_size{0};
|
||||
std::atomic<size_t> flushed_size{0};
|
||||
};
|
||||
};
|
||||
} // namespace td
|
78
tddb/td/db/utils/FileToStreamActor.cpp
Normal file
78
tddb/td/db/utils/FileToStreamActor.cpp
Normal file
|
@ -0,0 +1,78 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
#include "FileToStreamActor.h"
|
||||
|
||||
namespace td {
|
||||
FileToStreamActor::FileToStreamActor(FileFd fd, StreamWriter writer, Options options)
|
||||
: fd_(std::move(fd)), writer_(std::move(writer)), options_(options) {
|
||||
}
|
||||
|
||||
void FileToStreamActor::set_callback(td::unique_ptr<Callback> callback) {
|
||||
callback_ = std::move(callback);
|
||||
got_more();
|
||||
}
|
||||
|
||||
void FileToStreamActor::got_more() {
|
||||
if (!callback_) {
|
||||
return;
|
||||
}
|
||||
callback_->got_more();
|
||||
}
|
||||
void FileToStreamActor::loop() {
|
||||
auto dest = writer_.prepare_write();
|
||||
if (options_.limit != -1) {
|
||||
if (static_cast<int64>(dest.size()) > options_.limit) {
|
||||
dest.truncate(narrow_cast<size_t>(options_.limit));
|
||||
}
|
||||
}
|
||||
if (dest.empty()) {
|
||||
//NB: Owner of CyclicBufer::Reader should notify this actor after each chunk is readed
|
||||
return;
|
||||
}
|
||||
|
||||
auto r_size = fd_.read(dest);
|
||||
if (r_size.is_error()) {
|
||||
writer_.close_writer(r_size.move_as_error());
|
||||
got_more();
|
||||
return stop();
|
||||
}
|
||||
auto size = r_size.move_as_ok();
|
||||
writer_.confirm_write(size);
|
||||
got_more();
|
||||
if (options_.limit != -1) {
|
||||
options_.limit -= narrow_cast<int64>(size);
|
||||
}
|
||||
if (options_.limit == 0) {
|
||||
writer_.close_writer(td::Status::OK());
|
||||
got_more();
|
||||
return stop();
|
||||
}
|
||||
if (size == 0) {
|
||||
if (options_.read_tail_each < 0) {
|
||||
writer_.close_writer(td::Status::OK());
|
||||
got_more();
|
||||
return stop();
|
||||
}
|
||||
alarm_timestamp() = Timestamp::in(options_.read_tail_each);
|
||||
return;
|
||||
}
|
||||
yield();
|
||||
}
|
||||
|
||||
} // namespace td
|
53
tddb/td/db/utils/FileToStreamActor.h
Normal file
53
tddb/td/db/utils/FileToStreamActor.h
Normal file
|
@ -0,0 +1,53 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "StreamInterface.h"
|
||||
|
||||
#include "td/actor/actor.h"
|
||||
#include "td/utils/port/FileFd.h"
|
||||
|
||||
namespace td {
|
||||
class FileToStreamActor : public td::actor::Actor {
|
||||
public:
|
||||
struct Options {
|
||||
Options() {
|
||||
}
|
||||
int64 limit{-1};
|
||||
double read_tail_each{-1};
|
||||
};
|
||||
class Callback {
|
||||
public:
|
||||
virtual ~Callback() {
|
||||
}
|
||||
virtual void got_more() = 0;
|
||||
};
|
||||
FileToStreamActor(FileFd fd, StreamWriter writer, Options options = {});
|
||||
|
||||
void set_callback(td::unique_ptr<Callback> callback);
|
||||
|
||||
private:
|
||||
void got_more();
|
||||
void loop() override;
|
||||
FileFd fd_;
|
||||
StreamWriter writer_;
|
||||
td::unique_ptr<Callback> callback_;
|
||||
Options options_;
|
||||
};
|
||||
} // namespace td
|
79
tddb/td/db/utils/StreamInterface.cpp
Normal file
79
tddb/td/db/utils/StreamInterface.cpp
Normal file
|
@ -0,0 +1,79 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
#include "StreamInterface.h"
|
||||
namespace td {
|
||||
StreamReader::StreamReader(std::shared_ptr<StreamReaderInterface> self) : self(std::move(self)) {
|
||||
}
|
||||
size_t StreamReader::reader_size() {
|
||||
return self->reader_size();
|
||||
}
|
||||
Slice StreamReader::prepare_read() {
|
||||
return self->prepare_read();
|
||||
}
|
||||
Span<IoSlice> StreamReader::prepare_readv() {
|
||||
return self->prepare_readv();
|
||||
}
|
||||
void StreamReader::confirm_read(size_t size) {
|
||||
return self->confirm_read(size);
|
||||
}
|
||||
|
||||
void StreamReader::close_reader(Status error) {
|
||||
return self->close_reader(std::move(error));
|
||||
}
|
||||
bool StreamReader::is_writer_closed() const {
|
||||
return self->is_writer_closed();
|
||||
}
|
||||
Status &StreamReader::writer_status() {
|
||||
return self->writer_status();
|
||||
}
|
||||
|
||||
StreamWriter::StreamWriter(std::shared_ptr<StreamWriterInterface> self) : self(std::move(self)) {
|
||||
}
|
||||
size_t StreamWriter::writer_size() {
|
||||
return self->writer_size();
|
||||
}
|
||||
MutableSlice StreamWriter::prepare_write() {
|
||||
return self->prepare_write();
|
||||
}
|
||||
MutableSlice StreamWriter::prepare_write_at_least(size_t size) {
|
||||
return self->prepare_write_at_least(size);
|
||||
}
|
||||
void StreamWriter::confirm_write(size_t size) {
|
||||
return self->confirm_write(size);
|
||||
}
|
||||
void StreamWriter::append(Slice data) {
|
||||
return self->append(data);
|
||||
}
|
||||
void StreamWriter::append(BufferSlice data) {
|
||||
return self->append(std::move(data));
|
||||
}
|
||||
void StreamWriter::append(std::string data) {
|
||||
return self->append(std::move(data));
|
||||
}
|
||||
|
||||
void StreamWriter::close_writer(Status error) {
|
||||
return self->close_writer(std::move(error));
|
||||
}
|
||||
bool StreamWriter::is_reader_closed() const {
|
||||
return self->is_reader_closed();
|
||||
}
|
||||
Status &StreamWriter::reader_status() {
|
||||
return self->reader_status();
|
||||
}
|
||||
} // namespace td
|
102
tddb/td/db/utils/StreamInterface.h
Normal file
102
tddb/td/db/utils/StreamInterface.h
Normal file
|
@ -0,0 +1,102 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
#pragma once
|
||||
#include "td/utils/buffer.h"
|
||||
#include "td/utils/Slice.h"
|
||||
#include "td/utils/Span.h"
|
||||
#include "td/utils/port/IoSlice.h"
|
||||
|
||||
namespace td {
|
||||
// Generic stream interface
|
||||
// Will to hide implementations details.
|
||||
// CyclicBuffer, ChainBuffer, Bounded ChainBuffer, some clever writers. They all should be interchangable
|
||||
// Most implementaions will assume that reading and writing may happen concurrently
|
||||
|
||||
class StreamReaderInterface {
|
||||
public:
|
||||
virtual ~StreamReaderInterface() {
|
||||
}
|
||||
virtual size_t reader_size() = 0;
|
||||
virtual Slice prepare_read() = 0;
|
||||
virtual Span<IoSlice> prepare_readv() = 0;
|
||||
virtual void confirm_read(size_t size) = 0;
|
||||
|
||||
virtual void close_reader(Status error) = 0;
|
||||
virtual bool is_writer_closed() const = 0;
|
||||
virtual Status &writer_status() = 0;
|
||||
};
|
||||
|
||||
class StreamWriterInterface {
|
||||
public:
|
||||
virtual ~StreamWriterInterface() {
|
||||
}
|
||||
virtual size_t writer_size() = 0;
|
||||
virtual MutableSlice prepare_write() = 0;
|
||||
virtual MutableSlice prepare_write_at_least(size_t size) = 0;
|
||||
virtual void confirm_write(size_t size) = 0;
|
||||
virtual void append(Slice data) = 0;
|
||||
virtual void append(BufferSlice data) {
|
||||
append(data.as_slice());
|
||||
}
|
||||
virtual void append(std::string data) {
|
||||
append(Slice(data));
|
||||
}
|
||||
|
||||
virtual void close_writer(Status error) = 0;
|
||||
virtual bool is_reader_closed() const = 0;
|
||||
virtual Status &reader_status() = 0;
|
||||
};
|
||||
|
||||
// Hide shared_ptr
|
||||
class StreamReader : public StreamReaderInterface {
|
||||
public:
|
||||
StreamReader() = default;
|
||||
StreamReader(std::shared_ptr<StreamReaderInterface> self);
|
||||
size_t reader_size() override;
|
||||
Slice prepare_read() override;
|
||||
Span<IoSlice> prepare_readv() override;
|
||||
void confirm_read(size_t size) override;
|
||||
void close_reader(Status error) override;
|
||||
bool is_writer_closed() const override;
|
||||
Status &writer_status() override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<StreamReaderInterface> self;
|
||||
};
|
||||
|
||||
class StreamWriter : public StreamWriterInterface {
|
||||
public:
|
||||
StreamWriter() = default;
|
||||
StreamWriter(std::shared_ptr<StreamWriterInterface> self);
|
||||
size_t writer_size() override;
|
||||
MutableSlice prepare_write() override;
|
||||
MutableSlice prepare_write_at_least(size_t size) override;
|
||||
void confirm_write(size_t size) override;
|
||||
void append(Slice data) override;
|
||||
void append(BufferSlice data) override;
|
||||
void append(std::string data) override;
|
||||
void close_writer(Status error) override;
|
||||
bool is_reader_closed() const override;
|
||||
Status &reader_status() override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<StreamWriterInterface> self;
|
||||
};
|
||||
|
||||
} // namespace td
|
112
tddb/td/db/utils/StreamToFileActor.cpp
Normal file
112
tddb/td/db/utils/StreamToFileActor.cpp
Normal file
|
@ -0,0 +1,112 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
#include "StreamToFileActor.h"
|
||||
|
||||
namespace td {
|
||||
StreamToFileActor::StreamToFileActor(StreamReader reader, FileFd fd, FileSyncState::Writer sync_state, Options options)
|
||||
: reader_(std::move(reader)), fd_(std::move(fd)), sync_state_(std::move(sync_state)) {
|
||||
}
|
||||
void StreamToFileActor::set_callback(td::unique_ptr<Callback> callback) {
|
||||
callback_ = std::move(callback);
|
||||
callback_->on_sync_state_changed();
|
||||
}
|
||||
|
||||
Result<bool> StreamToFileActor::is_closed() {
|
||||
if (!reader_.is_writer_closed()) {
|
||||
return false;
|
||||
}
|
||||
return reader_.writer_status().clone();
|
||||
}
|
||||
|
||||
Status StreamToFileActor::do_flush_once() {
|
||||
auto size = reader_.reader_size();
|
||||
size_t total_written = 0;
|
||||
while (total_written < size) {
|
||||
auto io_slices = reader_.prepare_readv();
|
||||
TRY_RESULT(written, fd_.writev(io_slices));
|
||||
reader_.confirm_read(written);
|
||||
flushed_size_ += written;
|
||||
total_written += written;
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status StreamToFileActor::do_sync() {
|
||||
if (flushed_size_ == synced_size_) {
|
||||
return Status::OK();
|
||||
}
|
||||
TRY_STATUS(fd_.sync());
|
||||
synced_size_ = flushed_size_;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void StreamToFileActor::schedule_sync() {
|
||||
if (synced_size_ == flushed_size_) {
|
||||
return;
|
||||
}
|
||||
if (sync_state_.get_requested_synced_size() > synced_size_) {
|
||||
sync_at_.relax(Timestamp::in(options_.immediate_sync_delay));
|
||||
} else {
|
||||
sync_at_.relax(Timestamp::in(options_.lazy_sync_delay));
|
||||
}
|
||||
}
|
||||
|
||||
Result<bool> StreamToFileActor::do_loop() {
|
||||
// We must first check if writer is closed and then drain all data from reader
|
||||
// Otherwise there will be a race and some of data could be lost.
|
||||
// Also it could be useful to check error and stop immediately.
|
||||
TRY_RESULT(is_closed, is_closed());
|
||||
|
||||
// Flush all data that is awailable on the at the beginning of loop
|
||||
TRY_STATUS(do_flush_once());
|
||||
|
||||
if ((sync_at_ && sync_at_.is_in_past()) || is_closed) {
|
||||
TRY_STATUS(do_sync());
|
||||
sync_at_ = {};
|
||||
}
|
||||
|
||||
bool need_update = sync_state_.set_synced_size(synced_size_) | sync_state_.set_flushed_size(flushed_size_);
|
||||
if (need_update && callback_) {
|
||||
callback_->on_sync_state_changed();
|
||||
}
|
||||
|
||||
if (reader_.reader_size() == 0 && is_closed) {
|
||||
return true;
|
||||
}
|
||||
|
||||
schedule_sync();
|
||||
return false;
|
||||
}
|
||||
|
||||
void StreamToFileActor::start_up() {
|
||||
schedule_sync();
|
||||
}
|
||||
|
||||
void StreamToFileActor::loop() {
|
||||
auto r_is_closed = do_loop();
|
||||
if (r_is_closed.is_error()) {
|
||||
reader_.close_reader(r_is_closed.move_as_error());
|
||||
return stop();
|
||||
} else if (r_is_closed.ok()) {
|
||||
reader_.close_reader(Status::OK());
|
||||
return stop();
|
||||
}
|
||||
alarm_timestamp() = sync_at_;
|
||||
}
|
||||
} // namespace td
|
73
tddb/td/db/utils/StreamToFileActor.h
Normal file
73
tddb/td/db/utils/StreamToFileActor.h
Normal file
|
@ -0,0 +1,73 @@
|
|||
/*
|
||||
This file is part of TON Blockchain Library.
|
||||
|
||||
TON Blockchain Library is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
TON Blockchain Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Copyright 2017-2019 Telegram Systems LLP
|
||||
*/
|
||||
#pragma once
|
||||
#include "StreamInterface.h"
|
||||
#include "FileSyncState.h"
|
||||
|
||||
#include "td/utils/Time.h"
|
||||
#include "td/utils/port/FileFd.h"
|
||||
|
||||
#include "td/actor/actor.h"
|
||||
|
||||
namespace td {
|
||||
class StreamToFileActor : public actor::Actor {
|
||||
public:
|
||||
struct Options {
|
||||
Options() {
|
||||
}
|
||||
double lazy_sync_delay = 10;
|
||||
double immediate_sync_delay = 0.001;
|
||||
};
|
||||
|
||||
class Callback {
|
||||
public:
|
||||
virtual ~Callback() {
|
||||
}
|
||||
virtual void on_sync_state_changed() = 0;
|
||||
};
|
||||
|
||||
StreamToFileActor(StreamReader reader, FileFd fd, FileSyncState::Writer sync_state, Options options = {});
|
||||
void set_callback(td::unique_ptr<Callback> callback);
|
||||
|
||||
private:
|
||||
StreamReader reader_;
|
||||
FileFd fd_;
|
||||
Timestamp sync_at_;
|
||||
Options options_;
|
||||
FileSyncState::Writer sync_state_;
|
||||
unique_ptr<Callback> callback_;
|
||||
|
||||
size_t flushed_size_{0};
|
||||
size_t synced_size_{0};
|
||||
|
||||
TD_WARN_UNUSED_RESULT Result<bool> is_closed();
|
||||
|
||||
Status do_flush_once();
|
||||
|
||||
Status do_sync();
|
||||
|
||||
void schedule_sync();
|
||||
|
||||
TD_WARN_UNUSED_RESULT Result<bool> do_loop();
|
||||
|
||||
void start_up() override;
|
||||
|
||||
void loop() override;
|
||||
};
|
||||
} // namespace td
|
Loading…
Add table
Add a link
Reference in a new issue