external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

TON Storage utilities (#564)

Browse source 
* Rename chunk to piece in MerkleTree for consistency

* Refactor PeerManager

* Make PeerState thread-safe

* Download torrent by hash

* First version of storage daemon

* Download torrents partially

* Improve storing and loading torrent state in DB

* Rewrite MerkleTree

* "Remove torrent" in storage daemon

* Process errors, fix bugs in storage

* Move TonlibClientWrapper from rldp-http-proxy to tonlib

* Initial version of storage provider

* Move interaction with contracts to smc-util

* Improve TonlibClientWrapper interface

* Various improvements in storage provider

* Fix TorrentCreator.cpp

* Improve interface for partial download

* Client mode in storage-daemon

* Improve interface of storage-daemon-cli

* Fix calculating speed, show peers in storage-daemon

* Use permanent adnl id in storage daemon

* Fix sending large "storage.addUpdate" messages

* Improve printing torrents in cli

* Update tlo

* Fix RldpSender::on_ack

* Update storage provider

* Add "address" parameter to get-provider-params

* Allow client to close storage contract

* Limit torrent description

* Add more logs to storage provider

* smc.forget tonlib method

* Use smc.forget in storage daemon

* Optimize sending messages in smc-util.cpp

* Fix verbosity, remove excessive logs

* Json output in storage-daemon-cli

* Update storage provider contracts

* Fix rldp2 acks

* Change verbosity of logs in rldp2

* Update help and output of commands and in storage-daemon-cli

Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK 2022-12-22 12:24:13 +03:00 • committed by GitHub
parent 434dc487a4
commit 360ef54e6b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
75 changed files with 8872 additions and 1148 deletions
Download patch file Download diff file Expand all files Collapse all files

42
storage/TorrentCreator.cpp
View file

@ -25,11 +25,20 @@
#include "td/utils/PathView.h"
#include "td/utils/port/path.h"
#include "td/utils/tl_helpers.h"
#include "MicrochunkTree.h"
#include "TorrentHeader.hpp"
namespace ton {
td::Result<Torrent> Torrent::Creator::create_from_path(Options options, td::CSlice raw_path) {
TRY_RESULT(path, td::realpath(raw_path));
TRY_RESULT(stat, td::stat(path));
std::string root_dir = path;
while (!root_dir.empty() && root_dir.back() == TD_DIR_SLASH) {
root_dir.pop_back();
}
while (!root_dir.empty() && root_dir.back() != TD_DIR_SLASH) {
root_dir.pop_back();
}
if (stat.is_dir_) {
if (!path.empty() && path.back() != TD_DIR_SLASH) {
path += TD_DIR_SLASH;
@ -50,17 +59,21 @@ td::Result<Torrent> Torrent::Creator::create_from_path(Options options, td::CSli
});
TRY_STATUS(std::move(status));
TRY_STATUS(std::move(walk_status));
creator.root_dir_ = std::move(root_dir);
std::sort(creator.files_.begin(), creator.files_.end(),
[](const Torrent::Creator::File& a, const Torrent::Creator::File& b) { return a.name < b.name; });
return creator.finalize();
} else {
Torrent::Creator creator(options);
TRY_STATUS(creator.add_file(td::PathView(path).file_name(), path));
creator.root_dir_ = std::move(root_dir);
return creator.finalize();
}
}
td::Result<Torrent> Torrent::Creator::create_from_blobs(Options options, td::Span<Blob> blobs) {
Torrent::Creator creator(options);
for (auto &blob : blobs) {
for (auto& blob : blobs) {
TRY_STATUS(creator.add_blob(blob.name, blob.data));
}
return creator.finalize();
@ -79,7 +92,7 @@ td::Status Torrent::Creator::add_blob(td::Slice name, td::BlobView blob) {
}
TD_WARN_UNUSED_RESULT td::Status Torrent::Creator::add_file(td::Slice name, td::CSlice path) {
LOG(INFO) << "Add file " << name << " " << path;
LOG(DEBUG) << "Add file " << name << " " << path;
TRY_RESULT(data, td::FileNoCacheBlobView::create(path));
return add_blob(name, std::move(data));
}
@ -115,11 +128,9 @@ td::Result<Torrent> Torrent::Creator::finalize() {
auto header_size = header.serialization_size();
auto file_size = header_size + data_offset;
auto chunks_count = (file_size + options_.piece_size - 1) / options_.piece_size;
ton::MerkleTree tree;
tree.init_begin(chunks_count);
auto pieces_count = (file_size + options_.piece_size - 1) / options_.piece_size;
std::vector<Torrent::ChunkState> chunks;
size_t chunk_i = 0;
std::vector<td::Bits256> pieces;
auto flush_reader = [&](bool force) {
while (true) {
auto slice = reader.prepare_read();
@ -127,16 +138,14 @@ td::Result<Torrent> Torrent::Creator::finalize() {
if (slice.empty() || (slice.size() != options_.piece_size && !force)) {
break;
}
td::UInt256 hash;
td::Bits256 hash;
sha256(slice, hash.as_slice());
CHECK(chunk_i < chunks_count);
tree.init_add_chunk(chunk_i, hash.as_slice());
chunk_i++;
pieces.push_back(hash);
reader.confirm_read(slice.size());
}
};
td::uint64 offset = 0;
auto add_blob = [&](auto &&data, td::Slice name) {
auto add_blob = [&](auto data, td::Slice name) {
td::uint64 data_offset = 0;
while (data_offset < data.size()) {
auto dest = writer.prepare_write();
@ -168,24 +177,25 @@ td::Result<Torrent> Torrent::Creator::finalize() {
td::sha256(header_str, info.header_hash.as_slice());
add_blob(td::BufferSliceBlobView::create(td::BufferSlice(header_str)), "").ensure();
for (auto &file : files_) {
for (auto& file : files_) {
add_blob(std::move(file.data), file.name).ensure();
}
flush_reader(true);
tree.init_finish();
CHECK(chunk_i == chunks_count);
CHECK(pieces.size() == pieces_count);
CHECK(offset == file_size);
MerkleTree tree(std::move(pieces));
info.header_size = header.serialization_size();
info.piece_size = options_.piece_size;
info.description = options_.description;
info.file_size = file_size;
info.depth = tree.get_depth();
info.root_hash = tree.get_root_hash();
info.init_cell();
TRY_STATUS_PREFIX(info.validate(), "Invalid torrent info: ");
TRY_STATUS_PREFIX(header.validate(info.file_size, info.header_size), "Invalid torrent header: ");
Torrent torrent(info, std::move(header), std::move(tree), std::move(chunks));
Torrent torrent(info, std::move(header), std::move(tree), std::move(chunks), root_dir_);
return std::move(torrent);
}