mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
TON Storage utilities (#564)
* Rename chunk to piece in MerkleTree for consistency * Refactor PeerManager * Make PeerState thread-safe * Download torrent by hash * First version of storage daemon * Download torrents partially * Improve storing and loading torrent state in DB * Rewrite MerkleTree * "Remove torrent" in storage daemon * Process errors, fix bugs in storage * Move TonlibClientWrapper from rldp-http-proxy to tonlib * Initial version of storage provider * Move interaction with contracts to smc-util * Improve TonlibClientWrapper interface * Various improvements in storage provider * Fix TorrentCreator.cpp * Improve interface for partial download * Client mode in storage-daemon * Improve interface of storage-daemon-cli * Fix calculating speed, show peers in storage-daemon * Use permanent adnl id in storage daemon * Fix sending large "storage.addUpdate" messages * Improve printing torrents in cli * Update tlo * Fix RldpSender::on_ack * Update storage provider * Add "address" parameter to get-provider-params * Allow client to close storage contract * Limit torrent description * Add more logs to storage provider * smc.forget tonlib method * Use smc.forget in storage daemon * Optimize sending messages in smc-util.cpp * Fix verbosity, remove excessive logs * Json output in storage-daemon-cli * Update storage provider contracts * Fix rldp2 acks * Change verbosity of logs in rldp2 * Update help and output of commands and in storage-daemon-cli Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK
GitHub
parent
434dc487a4
commit
360ef54e6b
75 changed files with 8872 additions and 1148 deletions
|
|
@ -23,29 +23,30 @@
|
|||
#include "td/utils/crypto.h"
|
||||
#include "td/utils/port/Stat.h"
|
||||
#include "td/utils/tl_helpers.h"
|
||||
#include "td/utils/port/path.h"
|
||||
|
||||
namespace ton {
|
||||
|
||||
Torrent::Torrent(TorrentMeta meta)
|
||||
: info_(meta.info)
|
||||
, merkle_tree_(info_.pieces_count(), info_.root_hash)
|
||||
, piece_is_ready_(info_.pieces_count(), false) {
|
||||
not_ready_piece_count_ = piece_is_ready_.size();
|
||||
header_pieces_count_ = (info_.header_size + info_.piece_size - 1) / info_.piece_size;
|
||||
not_ready_pending_piece_count_ = header_pieces_count_;
|
||||
|
||||
if (meta.header) {
|
||||
set_header(meta.header.unwrap());
|
||||
} else {
|
||||
header_str_ = td::BufferSlice(info_.header_size);
|
||||
}
|
||||
if (meta.root_proof.not_null()) {
|
||||
merkle_tree_.add_proof(meta.root_proof);
|
||||
td::Result<Torrent> Torrent::open(Options options, td::Bits256 hash) {
|
||||
Torrent res(hash);
|
||||
if (!options.in_memory) {
|
||||
if (options.root_dir.empty()) {
|
||||
options.root_dir = ".";
|
||||
}
|
||||
res.set_root_dir(options.root_dir);
|
||||
}
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
td::Result<Torrent> Torrent::open(Options options, TorrentMeta meta) {
|
||||
Torrent res(std::move(meta));
|
||||
Torrent res(td::Bits256(meta.info.get_hash()));
|
||||
TRY_STATUS(res.init_info(std::move(meta.info)));
|
||||
if (meta.header) {
|
||||
TRY_STATUS(res.set_header(meta.header.unwrap()));
|
||||
}
|
||||
if (meta.root_proof.not_null()) {
|
||||
TRY_STATUS(res.merkle_tree_.add_proof(meta.root_proof));
|
||||
}
|
||||
if (!options.in_memory) {
|
||||
if (options.root_dir.empty()) {
|
||||
options.root_dir = ".";
|
||||
|
|
@ -64,6 +65,7 @@ td::Result<Torrent> Torrent::open(Options options, td::Slice meta_str) {
|
|||
}
|
||||
|
||||
const Torrent::Info &Torrent::get_info() const {
|
||||
CHECK(inited_info_);
|
||||
return info_;
|
||||
}
|
||||
|
||||
|
|
@ -103,6 +105,9 @@ td::Status Torrent::iterate_piece(Info::PieceInfo piece, F &&f) {
|
|||
}
|
||||
|
||||
bool Torrent::is_piece_ready(td::uint64 piece_i) const {
|
||||
if (!inited_info_) {
|
||||
return false;
|
||||
}
|
||||
CHECK(piece_i < info_.pieces_count());
|
||||
return piece_is_ready_[piece_i];
|
||||
}
|
||||
|
|
@ -133,7 +138,8 @@ Torrent::PartsRange Torrent::get_file_parts_range(size_t i) {
|
|||
return res;
|
||||
}
|
||||
|
||||
Torrent::PartsRange Torrent::get_header_parts_range() {
|
||||
Torrent::PartsRange Torrent::get_header_parts_range() const {
|
||||
CHECK(inited_info_);
|
||||
PartsRange res;
|
||||
res.begin = 0;
|
||||
res.end = header_pieces_count_;
|
||||
|
|
@ -184,11 +190,14 @@ std::string Torrent::get_stats_str() const {
|
|||
}
|
||||
|
||||
void Torrent::validate() {
|
||||
CHECK(header_);
|
||||
if (!inited_info_ || !header_) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::fill(piece_is_ready_.begin(), piece_is_ready_.end(), false);
|
||||
not_ready_piece_count_ = info_.pieces_count();
|
||||
|
||||
included_ready_size_ = 0;
|
||||
for (auto &chunk : chunks_) {
|
||||
chunk.ready_size = 0;
|
||||
if (root_dir_) {
|
||||
|
|
@ -200,31 +209,25 @@ void Torrent::validate() {
|
|||
}
|
||||
|
||||
std::vector<td::UInt256> hashes;
|
||||
std::vector<MerkleTree::Chunk> chunks;
|
||||
std::vector<std::pair<size_t, td::Bits256>> pieces;
|
||||
|
||||
auto flush = [&] {
|
||||
td::Bitset bitmask;
|
||||
merkle_tree_.add_chunks(chunks, bitmask);
|
||||
for (size_t i = 0; i < chunks.size(); i++) {
|
||||
if (!bitmask.get(i)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto piece_i = chunks[i].index;
|
||||
for (size_t piece_i : merkle_tree_.add_pieces(std::move(pieces))) {
|
||||
auto piece = info_.get_piece_info(piece_i);
|
||||
iterate_piece(piece, [&](auto it, auto info) {
|
||||
it->ready_size += info.size;
|
||||
if (!it->excluded) {
|
||||
included_ready_size_ += info.size;
|
||||
}
|
||||
return td::Status::OK();
|
||||
});
|
||||
piece_is_ready_[piece_i] = true;
|
||||
ready_parts_count_++;
|
||||
|
||||
CHECK(not_ready_piece_count_);
|
||||
not_ready_piece_count_--;
|
||||
}
|
||||
|
||||
hashes.clear();
|
||||
chunks.clear();
|
||||
pieces.clear();
|
||||
};
|
||||
|
||||
td::BufferSlice buf(info_.piece_size);
|
||||
|
|
@ -246,24 +249,23 @@ void Torrent::validate() {
|
|||
auto dest = buf.as_slice().truncate(info.size);
|
||||
TRY_STATUS(it->get_piece(dest, info.chunk_offset, &cache));
|
||||
sha256.feed(dest);
|
||||
//LOG(ERROR) << dest;
|
||||
return td::Status::OK();
|
||||
});
|
||||
if (is_ok.is_error()) {
|
||||
LOG_IF(ERROR, !skipped) << "Failed: " << is_ok;
|
||||
LOG(ERROR) << "Failed: " << is_ok;
|
||||
continue;
|
||||
}
|
||||
MerkleTree::Chunk chunk;
|
||||
chunk.index = piece_i;
|
||||
sha256.extract(chunk.hash.as_slice());
|
||||
|
||||
chunks.push_back(chunk);
|
||||
td::Bits256 hash;
|
||||
sha256.extract(hash.as_slice());
|
||||
pieces.emplace_back(piece_i, hash);
|
||||
}
|
||||
flush();
|
||||
}
|
||||
|
||||
td::Result<std::string> Torrent::get_piece_data(td::uint64 piece_i) {
|
||||
if (!inited_info_) {
|
||||
return td::Status::Error("Torrent info not inited");
|
||||
}
|
||||
CHECK(piece_i < info_.pieces_count());
|
||||
if (!piece_is_ready_[piece_i]) {
|
||||
return td::Status::Error("Piece is not ready");
|
||||
|
|
@ -272,6 +274,10 @@ td::Result<std::string> Torrent::get_piece_data(td::uint64 piece_i) {
|
|||
if (it != pending_pieces_.end()) {
|
||||
return it->second;
|
||||
}
|
||||
auto it2 = in_memory_pieces_.find(piece_i);
|
||||
if (it2 != in_memory_pieces_.end()) {
|
||||
return it2->second.data;
|
||||
}
|
||||
auto piece = info_.get_piece_info(piece_i);
|
||||
|
||||
std::string res(piece.size, '\0');
|
||||
|
|
@ -282,68 +288,114 @@ td::Result<std::string> Torrent::get_piece_data(td::uint64 piece_i) {
|
|||
}
|
||||
|
||||
td::Result<td::Ref<vm::Cell>> Torrent::get_piece_proof(td::uint64 piece_i) {
|
||||
if (!inited_info_) {
|
||||
return td::Status::Error("Torrent info not inited");
|
||||
}
|
||||
CHECK(piece_i < info_.pieces_count());
|
||||
return merkle_tree_.gen_proof(piece_i, piece_i);
|
||||
}
|
||||
|
||||
td::Status Torrent::add_piece(td::uint64 piece_i, td::Slice data, td::Ref<vm::Cell> proof) {
|
||||
TRY_STATUS(merkle_tree_.add_proof(proof));
|
||||
//LOG(ERROR) << "Add piece #" << piece_i;
|
||||
if (fatal_error_.is_error()) {
|
||||
return fatal_error_.clone().move_as_error_prefix("Fatal error: ");
|
||||
}
|
||||
if (!inited_info_) {
|
||||
return td::Status::Error("Torrent info not inited");
|
||||
}
|
||||
if (!proof.is_null()) {
|
||||
TRY_STATUS(merkle_tree_.add_proof(proof));
|
||||
}
|
||||
CHECK(piece_i < info_.pieces_count());
|
||||
if (piece_is_ready_[piece_i]) {
|
||||
return td::Status::OK();
|
||||
}
|
||||
td::Bits256 hash;
|
||||
td::sha256(data, hash.as_slice());
|
||||
TRY_RESULT(expected_hash, merkle_tree_.get_piece_hash(piece_i));
|
||||
if (expected_hash != hash) {
|
||||
return td::Status::Error("Hash mismatch");
|
||||
}
|
||||
piece_is_ready_[piece_i] = true;
|
||||
ready_parts_count_++;
|
||||
ton::MerkleTree::Chunk chunk;
|
||||
chunk.index = piece_i;
|
||||
td::sha256(data, chunk.hash.as_slice());
|
||||
TRY_STATUS(merkle_tree_.try_add_chunks({chunk}));
|
||||
|
||||
if (chunks_.empty()) {
|
||||
return add_header_piece(piece_i, data);
|
||||
if (chunks_.empty() || !enabled_wirte_to_files_) {
|
||||
return add_pending_piece(piece_i, data);
|
||||
}
|
||||
|
||||
return add_validated_piece(piece_i, data);
|
||||
}
|
||||
|
||||
td::Status Torrent::add_header_piece(td::uint64 piece_i, td::Slice data) {
|
||||
td::Status Torrent::add_proof(td::Ref<vm::Cell> proof) {
|
||||
if (!inited_info_) {
|
||||
return td::Status::Error("Torrent info not inited");
|
||||
}
|
||||
return merkle_tree_.add_proof(std::move(proof));
|
||||
}
|
||||
|
||||
td::Status Torrent::add_pending_piece(td::uint64 piece_i, td::Slice data) {
|
||||
pending_pieces_[piece_i] = data.str();
|
||||
|
||||
if (piece_i < header_pieces_count_) {
|
||||
//LOG(ERROR) << "Add header piece #" << piece_i;
|
||||
auto piece = info_.get_piece_info(piece_i);
|
||||
auto dest = header_str_.as_slice().substr(piece.offset);
|
||||
data.truncate(dest.size());
|
||||
dest.copy_from(data);
|
||||
not_ready_pending_piece_count_--;
|
||||
if (not_ready_pending_piece_count_ == 0) {
|
||||
//LOG(ERROR) << "Got full header";
|
||||
TorrentHeader header;
|
||||
TRY_STATUS(td::unserialize(header, header_str_.as_slice())); // TODO: it is a fatal error
|
||||
set_header(header);
|
||||
for (auto &it : pending_pieces_) {
|
||||
TRY_STATUS(add_validated_piece(it.first, it.second));
|
||||
auto S = td::unserialize(header, header_str_.as_slice());
|
||||
if (S.is_ok()) {
|
||||
S = set_header(std::move(header));
|
||||
}
|
||||
if (S.is_error()) {
|
||||
S = S.move_as_error_prefix("Invalid torrent header: ");
|
||||
fatal_error_ = S.clone();
|
||||
return S;
|
||||
}
|
||||
if (enabled_wirte_to_files_) {
|
||||
add_pending_pieces();
|
||||
}
|
||||
pending_pieces_.clear();
|
||||
}
|
||||
} else {
|
||||
LOG(ERROR) << "PENDING";
|
||||
}
|
||||
|
||||
return td::Status::OK();
|
||||
}
|
||||
|
||||
std::string Torrent::get_chunk_path(td::Slice name) {
|
||||
void Torrent::enable_write_to_files() {
|
||||
if (enabled_wirte_to_files_) {
|
||||
return;
|
||||
}
|
||||
enabled_wirte_to_files_ = true;
|
||||
if (header_) {
|
||||
add_pending_pieces();
|
||||
}
|
||||
}
|
||||
|
||||
void Torrent::add_pending_pieces() {
|
||||
for (auto &p : pending_pieces_) {
|
||||
td::Status S = add_validated_piece(p.first, std::move(p.second));
|
||||
if (S.is_error()) {
|
||||
LOG(WARNING) << "Failed to add pending piece #" << p.first << ": " << S;
|
||||
}
|
||||
}
|
||||
pending_pieces_.clear();
|
||||
}
|
||||
|
||||
std::string Torrent::get_chunk_path(td::Slice name) const {
|
||||
return PSTRING() << root_dir_.value() << TD_DIR_SLASH << header_.value().dir_name << TD_DIR_SLASH << name;
|
||||
}
|
||||
|
||||
std::string Torrent::get_file_path(size_t i) const {
|
||||
return get_chunk_path(chunks_.at(i + 1).name);
|
||||
}
|
||||
|
||||
td::Status Torrent::init_chunk_data(ChunkState &chunk) {
|
||||
if (chunk.data) {
|
||||
return td::Status::OK();
|
||||
}
|
||||
if (root_dir_) {
|
||||
TRY_RESULT(data, td::FileNoCacheBlobView::create(get_chunk_path(chunk.name), chunk.size, true));
|
||||
std::string path = get_chunk_path(chunk.name);
|
||||
TRY_STATUS(td::mkpath(path));
|
||||
TRY_RESULT(data, td::FileNoCacheBlobView::create(path, chunk.size, true));
|
||||
chunk.data = std::move(data);
|
||||
} else {
|
||||
chunk.data = td::BufferSliceBlobView::create(td::BufferSlice(chunk.size));
|
||||
|
|
@ -355,22 +407,41 @@ td::Status Torrent::add_validated_piece(td::uint64 piece_i, td::Slice data) {
|
|||
CHECK(!chunks_.empty());
|
||||
|
||||
auto piece = info_.get_piece_info(piece_i);
|
||||
std::set<size_t> excluded;
|
||||
|
||||
TRY_STATUS(iterate_piece(piece, [&](auto it, auto info) {
|
||||
if (it->excluded) {
|
||||
excluded.insert(it - chunks_.begin());
|
||||
return td::Status::OK();
|
||||
}
|
||||
TRY_STATUS(init_chunk_data(*it));
|
||||
return it->add_piece(data.substr(info.piece_offset, info.size), info.chunk_offset);
|
||||
TRY_STATUS(it->write_piece(data.substr(info.piece_offset, info.size), info.chunk_offset));
|
||||
return td::Status::OK();
|
||||
}));
|
||||
TRY_STATUS(iterate_piece(piece, [&](auto it, auto info) {
|
||||
if (!it->excluded) {
|
||||
it->ready_size += info.size;
|
||||
included_ready_size_ += info.size;
|
||||
}
|
||||
return td::Status::OK();
|
||||
}));
|
||||
piece_is_ready_[piece_i] = true;
|
||||
not_ready_piece_count_--;
|
||||
if (!excluded.empty()) {
|
||||
in_memory_pieces_[piece_i] = {data.str(), std::move(excluded)};
|
||||
}
|
||||
|
||||
return td::Status::OK();
|
||||
}
|
||||
|
||||
bool Torrent::is_completed() const {
|
||||
return not_ready_piece_count_ == 0;
|
||||
return inited_info_ && enabled_wirte_to_files_ && included_ready_size_ == included_size_;
|
||||
}
|
||||
|
||||
td::Result<td::BufferSlice> Torrent::read_file(td::Slice name) {
|
||||
if (!inited_info_) {
|
||||
return td::Status::Error("Torrent info not inited");
|
||||
}
|
||||
for (auto &chunk : chunks_) {
|
||||
if (chunk.name == name) {
|
||||
td::BufferSlice res(chunk.size);
|
||||
|
|
@ -383,10 +454,12 @@ td::Result<td::BufferSlice> Torrent::read_file(td::Slice name) {
|
|||
|
||||
Torrent::GetMetaOptions::GetMetaOptions() = default;
|
||||
std::string Torrent::get_meta_str(const GetMetaOptions &options) const {
|
||||
CHECK(inited_info_);
|
||||
return get_meta(options).serialize();
|
||||
}
|
||||
|
||||
TorrentMeta Torrent::get_meta(const GetMetaOptions &options) const {
|
||||
CHECK(inited_info_);
|
||||
TorrentMeta torrent_file;
|
||||
if (options.with_header) {
|
||||
torrent_file.header = header_;
|
||||
|
|
@ -399,17 +472,36 @@ TorrentMeta Torrent::get_meta(const GetMetaOptions &options) const {
|
|||
return torrent_file;
|
||||
}
|
||||
|
||||
Torrent::Torrent(Info info, td::optional<TorrentHeader> header, ton::MerkleTree tree, std::vector<ChunkState> chunks)
|
||||
: info_(info)
|
||||
Torrent::Torrent(td::Bits256 hash) : hash_(hash), inited_info_(false) {
|
||||
}
|
||||
|
||||
Torrent::Torrent(Info info, td::optional<TorrentHeader> header, ton::MerkleTree tree, std::vector<ChunkState> chunks,
|
||||
std::string root_dir)
|
||||
: hash_(info.get_hash())
|
||||
, inited_info_(true)
|
||||
, info_(info)
|
||||
, root_dir_(std::move(root_dir))
|
||||
, header_(std::move(header))
|
||||
, enabled_wirte_to_files_(true)
|
||||
, merkle_tree_(std::move(tree))
|
||||
, piece_is_ready_(info_.pieces_count(), true)
|
||||
, ready_parts_count_{info_.pieces_count()}
|
||||
, chunks_(std::move(chunks)) {
|
||||
, chunks_(std::move(chunks))
|
||||
, included_size_(info_.file_size)
|
||||
, included_ready_size_(info_.file_size) {
|
||||
}
|
||||
|
||||
void Torrent::set_header(const TorrentHeader &header) {
|
||||
header_ = header;
|
||||
td::Status Torrent::set_header(TorrentHeader header) {
|
||||
if (header_) {
|
||||
return td::Status::OK();
|
||||
}
|
||||
auto header_str = header.serialize();
|
||||
td::Bits256 header_hash;
|
||||
td::sha256(header_str.as_slice(), header_hash.as_slice());
|
||||
if (header_hash != info_.header_hash) {
|
||||
return td::Status::Error("Incorrect header hash");
|
||||
}
|
||||
TRY_STATUS_PREFIX(header.validate(info_.file_size, info_.header_size), "Invalid torrent header: ");
|
||||
auto add_chunk = [&](td::Slice name, td::uint64 offset, td::uint64 size) {
|
||||
ChunkState chunk;
|
||||
chunk.name = name.str();
|
||||
|
|
@ -417,14 +509,17 @@ void Torrent::set_header(const TorrentHeader &header) {
|
|||
chunk.size = size;
|
||||
chunk.offset = offset;
|
||||
chunks_.push_back(std::move(chunk));
|
||||
included_size_ += size;
|
||||
};
|
||||
add_chunk("", 0, header.serialization_size());
|
||||
chunks_.back().data = td::BufferSliceBlobView::create(header.serialize());
|
||||
add_chunk("", 0, header_str.size());
|
||||
chunks_.back().data = td::BufferSliceBlobView::create(std::move(header_str));
|
||||
for (size_t i = 0; i < header.files_count; i++) {
|
||||
auto l = header.get_data_begin(i);
|
||||
auto r = header.get_data_end(i);
|
||||
add_chunk(header.get_name(i), l, r - l);
|
||||
}
|
||||
header_ = std::move(header);
|
||||
return td::Status::OK();
|
||||
}
|
||||
|
||||
size_t Torrent::get_ready_parts_count() const {
|
||||
|
|
@ -432,6 +527,7 @@ size_t Torrent::get_ready_parts_count() const {
|
|||
}
|
||||
|
||||
std::vector<size_t> Torrent::chunks_by_piece(td::uint64 piece_id) {
|
||||
CHECK(inited_info_);
|
||||
std::vector<size_t> res;
|
||||
auto piece = info_.get_piece_info(piece_id);
|
||||
auto is_ok = iterate_piece(piece, [&](auto it, auto info) {
|
||||
|
|
@ -441,4 +537,165 @@ std::vector<size_t> Torrent::chunks_by_piece(td::uint64 piece_id) {
|
|||
return res;
|
||||
}
|
||||
|
||||
td::Status Torrent::init_info(Info info) {
|
||||
if (hash_ != info.get_hash()) {
|
||||
return td::Status::Error("Hash mismatch");
|
||||
}
|
||||
if (inited_info_) {
|
||||
return td::Status::OK();
|
||||
}
|
||||
auto S = info.validate();
|
||||
if (S.is_error()) {
|
||||
S = S.move_as_error_prefix("Invalid torrent info: ");
|
||||
fatal_error_ = S.clone();
|
||||
return S;
|
||||
}
|
||||
inited_info_ = true;
|
||||
info_ = std::move(info);
|
||||
merkle_tree_ = MerkleTree(info_.pieces_count(), info_.root_hash);
|
||||
piece_is_ready_.resize(info_.pieces_count(), false);
|
||||
not_ready_piece_count_ = piece_is_ready_.size();
|
||||
header_pieces_count_ = (info_.header_size + info_.piece_size - 1) / info_.piece_size;
|
||||
not_ready_pending_piece_count_ = header_pieces_count_;
|
||||
header_str_ = td::BufferSlice(info_.header_size);
|
||||
return td::Status::OK();
|
||||
}
|
||||
|
||||
void Torrent::set_file_excluded(size_t i, bool excluded) {
|
||||
CHECK(header_);
|
||||
CHECK(i + 1 < chunks_.size());
|
||||
if (!root_dir_) {
|
||||
return; // All files are in-memory, nothing to do
|
||||
}
|
||||
size_t chunk_i = i + 1;
|
||||
auto &chunk = chunks_[chunk_i];
|
||||
if (chunk.excluded == excluded) {
|
||||
return;
|
||||
}
|
||||
if (excluded) {
|
||||
included_size_ -= chunk.size;
|
||||
included_ready_size_ -= chunk.ready_size;
|
||||
} else {
|
||||
included_size_ += chunk.size;
|
||||
included_ready_size_ += chunk.ready_size;
|
||||
}
|
||||
chunk.excluded = excluded;
|
||||
if (!enabled_wirte_to_files_ || excluded) {
|
||||
return;
|
||||
}
|
||||
auto range = get_file_parts_range(i);
|
||||
for (auto it = in_memory_pieces_.lower_bound(range.begin); it != in_memory_pieces_.end() && it->first < range.end;) {
|
||||
if (!it->second.pending_chunks.count(chunk_i)) {
|
||||
++it;
|
||||
continue;
|
||||
}
|
||||
auto piece_i = it->first;
|
||||
auto piece = info_.get_piece_info(piece_i);
|
||||
auto S = [&]() {
|
||||
auto l = td::max(chunk.offset, piece.offset);
|
||||
auto r = td::min(chunk.offset + chunk.size, piece.offset + piece.size);
|
||||
TRY_STATUS(init_chunk_data(chunk));
|
||||
TRY_STATUS(chunk.write_piece(it->second.data.substr(l - piece.offset, r - l), l - chunk.offset));
|
||||
chunk.ready_size += r - l;
|
||||
included_ready_size_ += r - l;
|
||||
return td::Status::OK();
|
||||
}();
|
||||
if (S.is_error()) {
|
||||
// Erase piece completely
|
||||
piece_is_ready_[piece_i] = false;
|
||||
not_ready_piece_count_++;
|
||||
iterate_piece(piece, [&](auto it2, auto info) {
|
||||
if (!it2->excluded) {
|
||||
included_ready_size_ -= info.size;
|
||||
}
|
||||
if (!it2->excluded || !it->second.pending_chunks.count(it2 - chunks_.begin())) {
|
||||
it2->ready_size -= info.size;
|
||||
}
|
||||
return td::Status::OK();
|
||||
});
|
||||
it = in_memory_pieces_.erase(it);
|
||||
continue;
|
||||
}
|
||||
it->second.pending_chunks.erase(chunk_i);
|
||||
if (it->second.pending_chunks.empty()) {
|
||||
it = in_memory_pieces_.erase(it);
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Torrent::load_from_files(std::string files_path) {
|
||||
CHECK(inited_header());
|
||||
std::vector<td::optional<td::BlobView>> new_blobs;
|
||||
new_blobs.push_back(td::BufferSliceBlobView::create(get_header().serialize()));
|
||||
size_t files_count = get_files_count().unwrap();
|
||||
for (size_t i = 0; i < files_count; ++i) {
|
||||
std::string new_path = PSTRING() << files_path << TD_DIR_SLASH << header_.value().dir_name << TD_DIR_SLASH
|
||||
<< get_file_name(i);
|
||||
auto R = td::FileNoCacheBlobView::create(new_path, get_file_size(i), false);
|
||||
if (R.is_error() && files_count == 1) {
|
||||
R = td::FileNoCacheBlobView::create(files_path, get_file_size(i), false);
|
||||
}
|
||||
if (R.is_error()) {
|
||||
new_blobs.emplace_back();
|
||||
} else {
|
||||
new_blobs.push_back(R.move_as_ok());
|
||||
}
|
||||
}
|
||||
auto load_new_piece = [&](size_t piece_i) -> td::Result<std::string> {
|
||||
auto piece = info_.get_piece_info(piece_i);
|
||||
bool included = false;
|
||||
TRY_STATUS(iterate_piece(piece, [&](auto it, IterateInfo info) {
|
||||
if (!it->excluded) {
|
||||
included = true;
|
||||
}
|
||||
return td::Status::OK();
|
||||
}));
|
||||
if (!included) {
|
||||
return td::Status::Error("Piece is excluded");
|
||||
}
|
||||
std::string data(piece.size, '\0');
|
||||
TRY_STATUS(iterate_piece(piece, [&](auto it, IterateInfo info) {
|
||||
size_t chunk_i = it - chunks_.begin();
|
||||
if (!new_blobs[chunk_i]) {
|
||||
return td::Status::Error("No such file");
|
||||
}
|
||||
TRY_RESULT(s, new_blobs[chunk_i].value().view_copy(td::MutableSlice(data).substr(info.piece_offset, info.size),
|
||||
info.chunk_offset));
|
||||
if (s != info.size) {
|
||||
return td::Status::Error("Can't read file");
|
||||
}
|
||||
return td::Status::OK();
|
||||
}));
|
||||
return data;
|
||||
};
|
||||
std::vector<std::pair<size_t, td::Bits256>> new_pieces;
|
||||
for (size_t i = 0; i < piece_is_ready_.size(); ++i) {
|
||||
if (piece_is_ready_[i]) {
|
||||
continue;
|
||||
}
|
||||
auto r_data = load_new_piece(i);
|
||||
if (r_data.is_error()) {
|
||||
continue;
|
||||
}
|
||||
td::Bits256 hash;
|
||||
td::sha256(r_data.ok(), hash.as_slice());
|
||||
new_pieces.emplace_back(i, hash);
|
||||
}
|
||||
size_t added_cnt = 0;
|
||||
for (size_t i : merkle_tree_.add_pieces(std::move(new_pieces))) {
|
||||
auto r_data = load_new_piece(i);
|
||||
if (r_data.is_error()) {
|
||||
continue;
|
||||
}
|
||||
if (add_piece(i, r_data.ok(), {}).is_ok()) {
|
||||
++added_cnt;
|
||||
}
|
||||
}
|
||||
if (added_cnt > 0) {
|
||||
LOG(INFO) << "Loaded " << added_cnt << " new pieces for " << get_hash().to_hex();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace ton
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue