2020-07-06 14:07:20 +00:00
|
|
|
/*
|
|
|
|
This file is part of TON Blockchain Library.
|
|
|
|
|
|
|
|
TON Blockchain Library is free software: you can redistribute it and/or modify
|
|
|
|
it under the terms of the GNU Lesser General Public License as published by
|
|
|
|
the Free Software Foundation, either version 2 of the License, or
|
|
|
|
(at your option) any later version.
|
|
|
|
|
|
|
|
TON Blockchain Library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
GNU Lesser General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU Lesser General Public License
|
|
|
|
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
Copyright 2017-2020 Telegram Systems LLP
|
|
|
|
*/
|
|
|
|
|
2020-05-27 18:10:46 +00:00
|
|
|
#include "TorrentCreator.h"
|
|
|
|
|
|
|
|
#include "td/db/utils/CyclicBuffer.h"
|
|
|
|
|
|
|
|
#include "td/utils/crypto.h"
|
|
|
|
#include "td/utils/PathView.h"
|
|
|
|
#include "td/utils/port/path.h"
|
|
|
|
#include "td/utils/tl_helpers.h"
|
2022-12-22 09:24:13 +00:00
|
|
|
#include "MicrochunkTree.h"
|
|
|
|
#include "TorrentHeader.hpp"
|
2020-05-27 18:10:46 +00:00
|
|
|
|
|
|
|
namespace ton {
|
2023-04-07 12:50:07 +00:00
|
|
|
static bool is_dir_slash(char c) {
|
|
|
|
return (c == TD_DIR_SLASH) | (c == '/');
|
|
|
|
}
|
|
|
|
|
2020-05-27 18:10:46 +00:00
|
|
|
td::Result<Torrent> Torrent::Creator::create_from_path(Options options, td::CSlice raw_path) {
|
|
|
|
TRY_RESULT(path, td::realpath(raw_path));
|
|
|
|
TRY_RESULT(stat, td::stat(path));
|
2022-12-22 09:24:13 +00:00
|
|
|
std::string root_dir = path;
|
2023-04-07 12:50:07 +00:00
|
|
|
while (!root_dir.empty() && is_dir_slash(root_dir.back())) {
|
2022-12-22 09:24:13 +00:00
|
|
|
root_dir.pop_back();
|
|
|
|
}
|
2023-04-07 12:50:07 +00:00
|
|
|
while (!root_dir.empty() && !is_dir_slash(root_dir.back())) {
|
2022-12-22 09:24:13 +00:00
|
|
|
root_dir.pop_back();
|
|
|
|
}
|
2020-05-27 18:10:46 +00:00
|
|
|
if (stat.is_dir_) {
|
2023-04-07 12:50:07 +00:00
|
|
|
if (!path.empty() && !is_dir_slash(path.back())) {
|
2020-05-27 18:10:46 +00:00
|
|
|
path += TD_DIR_SLASH;
|
|
|
|
}
|
|
|
|
if (!options.dir_name) {
|
|
|
|
options.dir_name = td::PathView::dir_and_file(path).str();
|
|
|
|
}
|
|
|
|
Torrent::Creator creator(options);
|
|
|
|
td::Status status;
|
|
|
|
auto walk_status = td::WalkPath::run(path, [&](td::CSlice name, td::WalkPath::Type type) {
|
|
|
|
if (type == td::WalkPath::Type::NotDir) {
|
2023-04-07 12:50:07 +00:00
|
|
|
std::string rel_name = td::PathView::relative(name, path).str();
|
|
|
|
td::Slice file_name = rel_name;
|
|
|
|
for (size_t i = 0; i < rel_name.size(); ++i) {
|
|
|
|
if (is_dir_slash(rel_name[i])) {
|
|
|
|
rel_name[i] = '/';
|
|
|
|
file_name = td::Slice(rel_name.c_str() + i + 1, rel_name.c_str() + rel_name.size());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Exclude OS-created files that can be modified automatically and thus break some torrent pieces
|
|
|
|
if (file_name == ".DS_Store" || td::to_lower(file_name) == "desktop.ini" ||
|
|
|
|
td::to_lower(file_name) == "thumbs.db") {
|
|
|
|
return td::WalkPath::Action::Continue;
|
|
|
|
}
|
|
|
|
status = creator.add_file(rel_name, name);
|
2020-05-27 18:10:46 +00:00
|
|
|
if (status.is_error()) {
|
|
|
|
return td::WalkPath::Action::Abort;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return td::WalkPath::Action::Continue;
|
|
|
|
});
|
|
|
|
TRY_STATUS(std::move(status));
|
|
|
|
TRY_STATUS(std::move(walk_status));
|
2022-12-22 09:24:13 +00:00
|
|
|
creator.root_dir_ = std::move(root_dir);
|
|
|
|
std::sort(creator.files_.begin(), creator.files_.end(),
|
|
|
|
[](const Torrent::Creator::File& a, const Torrent::Creator::File& b) { return a.name < b.name; });
|
2020-05-27 18:10:46 +00:00
|
|
|
return creator.finalize();
|
|
|
|
} else {
|
|
|
|
Torrent::Creator creator(options);
|
|
|
|
TRY_STATUS(creator.add_file(td::PathView(path).file_name(), path));
|
2022-12-22 09:24:13 +00:00
|
|
|
creator.root_dir_ = std::move(root_dir);
|
2020-05-27 18:10:46 +00:00
|
|
|
return creator.finalize();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
td::Result<Torrent> Torrent::Creator::create_from_blobs(Options options, td::Span<Blob> blobs) {
|
|
|
|
Torrent::Creator creator(options);
|
2022-12-22 09:24:13 +00:00
|
|
|
for (auto& blob : blobs) {
|
2020-05-27 18:10:46 +00:00
|
|
|
TRY_STATUS(creator.add_blob(blob.name, blob.data));
|
|
|
|
}
|
|
|
|
return creator.finalize();
|
|
|
|
}
|
|
|
|
|
|
|
|
td::Status Torrent::Creator::add_blob(td::Slice name, td::Slice blob) {
|
|
|
|
return add_blob(name, td::BufferSliceBlobView::create(td::BufferSlice(blob)));
|
|
|
|
}
|
|
|
|
|
|
|
|
td::Status Torrent::Creator::add_blob(td::Slice name, td::BlobView blob) {
|
|
|
|
File file;
|
|
|
|
file.name = name.str();
|
|
|
|
file.data = std::move(blob);
|
|
|
|
files_.push_back(std::move(file));
|
|
|
|
return td::Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
TD_WARN_UNUSED_RESULT td::Status Torrent::Creator::add_file(td::Slice name, td::CSlice path) {
|
2022-12-22 09:24:13 +00:00
|
|
|
LOG(DEBUG) << "Add file " << name << " " << path;
|
2020-05-27 18:10:46 +00:00
|
|
|
TRY_RESULT(data, td::FileNoCacheBlobView::create(path));
|
|
|
|
return add_blob(name, std::move(data));
|
|
|
|
}
|
|
|
|
|
|
|
|
td::Result<Torrent> Torrent::Creator::finalize() {
|
2022-12-24 10:10:09 +00:00
|
|
|
if (files_.empty()) {
|
|
|
|
return td::Status::Error("No files");
|
|
|
|
}
|
2020-05-27 18:10:46 +00:00
|
|
|
TorrentHeader header;
|
|
|
|
TRY_RESULT(files_count, td::narrow_cast_safe<td::uint32>(files_.size()));
|
|
|
|
header.files_count = files_count;
|
|
|
|
header.data_index.resize(files_count);
|
|
|
|
header.name_index.resize(files_count);
|
|
|
|
td::uint64 data_offset = 0;
|
|
|
|
for (size_t i = 0; i < files_.size(); i++) {
|
|
|
|
header.names += files_[i].name;
|
|
|
|
header.name_index[i] = header.names.size();
|
|
|
|
data_offset += files_[i].data.size();
|
|
|
|
header.data_index[i] = data_offset;
|
|
|
|
}
|
|
|
|
header.tot_names_size = header.names.size();
|
|
|
|
if (options_.dir_name) {
|
|
|
|
header.dir_name = options_.dir_name.value();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Now we should stream all data to calculate sha256 of all pieces
|
|
|
|
|
|
|
|
std::string buffer;
|
|
|
|
td::CyclicBuffer::Options cb_options;
|
|
|
|
cb_options.chunk_size = td::max(options_.piece_size * 16, (1 << 20) / options_.piece_size * options_.piece_size);
|
|
|
|
cb_options.count = 2;
|
|
|
|
|
|
|
|
auto reader_writer = td::CyclicBuffer::create(cb_options);
|
|
|
|
auto reader = std::move(reader_writer.first);
|
|
|
|
auto writer = std::move(reader_writer.second);
|
|
|
|
|
|
|
|
auto header_size = header.serialization_size();
|
|
|
|
auto file_size = header_size + data_offset;
|
2022-12-22 09:24:13 +00:00
|
|
|
auto pieces_count = (file_size + options_.piece_size - 1) / options_.piece_size;
|
2020-05-27 18:10:46 +00:00
|
|
|
std::vector<Torrent::ChunkState> chunks;
|
2022-12-22 09:24:13 +00:00
|
|
|
std::vector<td::Bits256> pieces;
|
2020-05-27 18:10:46 +00:00
|
|
|
auto flush_reader = [&](bool force) {
|
|
|
|
while (true) {
|
|
|
|
auto slice = reader.prepare_read();
|
|
|
|
slice.truncate(options_.piece_size);
|
|
|
|
if (slice.empty() || (slice.size() != options_.piece_size && !force)) {
|
|
|
|
break;
|
|
|
|
}
|
2022-12-22 09:24:13 +00:00
|
|
|
td::Bits256 hash;
|
2020-05-27 18:10:46 +00:00
|
|
|
sha256(slice, hash.as_slice());
|
2022-12-22 09:24:13 +00:00
|
|
|
pieces.push_back(hash);
|
2020-05-27 18:10:46 +00:00
|
|
|
reader.confirm_read(slice.size());
|
|
|
|
}
|
|
|
|
};
|
|
|
|
td::uint64 offset = 0;
|
2022-12-22 09:24:13 +00:00
|
|
|
auto add_blob = [&](auto data, td::Slice name) {
|
2020-05-27 18:10:46 +00:00
|
|
|
td::uint64 data_offset = 0;
|
|
|
|
while (data_offset < data.size()) {
|
|
|
|
auto dest = writer.prepare_write();
|
|
|
|
CHECK(dest.size() != 0);
|
|
|
|
dest.truncate(data.size() - data_offset);
|
|
|
|
TRY_RESULT(got_size, data.view_copy(dest, data_offset));
|
|
|
|
CHECK(got_size != 0);
|
|
|
|
data_offset += got_size;
|
|
|
|
writer.confirm_write(got_size);
|
|
|
|
flush_reader(false);
|
|
|
|
}
|
|
|
|
|
|
|
|
Torrent::ChunkState chunk;
|
|
|
|
chunk.name = name.str();
|
|
|
|
chunk.offset = offset;
|
|
|
|
chunk.size = data.size();
|
|
|
|
chunk.ready_size = chunk.size;
|
|
|
|
chunk.data = std::move(data);
|
|
|
|
|
|
|
|
offset += chunk.size;
|
|
|
|
chunks.push_back(std::move(chunk));
|
|
|
|
return td::Status::OK();
|
|
|
|
};
|
|
|
|
|
|
|
|
Torrent::Info info;
|
|
|
|
auto header_str = td::serialize(header);
|
|
|
|
CHECK(header_size == header_str.size());
|
|
|
|
info.header_size = header_str.size();
|
|
|
|
td::sha256(header_str, info.header_hash.as_slice());
|
|
|
|
|
|
|
|
add_blob(td::BufferSliceBlobView::create(td::BufferSlice(header_str)), "").ensure();
|
2022-12-22 09:24:13 +00:00
|
|
|
for (auto& file : files_) {
|
2020-05-27 18:10:46 +00:00
|
|
|
add_blob(std::move(file.data), file.name).ensure();
|
|
|
|
}
|
|
|
|
flush_reader(true);
|
2022-12-22 09:24:13 +00:00
|
|
|
CHECK(pieces.size() == pieces_count);
|
2020-05-27 18:10:46 +00:00
|
|
|
CHECK(offset == file_size);
|
2022-12-22 09:24:13 +00:00
|
|
|
MerkleTree tree(std::move(pieces));
|
2020-05-27 18:10:46 +00:00
|
|
|
|
|
|
|
info.header_size = header.serialization_size();
|
|
|
|
info.piece_size = options_.piece_size;
|
|
|
|
info.description = options_.description;
|
|
|
|
info.file_size = file_size;
|
|
|
|
info.root_hash = tree.get_root_hash();
|
|
|
|
|
|
|
|
info.init_cell();
|
2022-12-22 09:24:13 +00:00
|
|
|
TRY_STATUS_PREFIX(info.validate(), "Invalid torrent info: ");
|
|
|
|
TRY_STATUS_PREFIX(header.validate(info.file_size, info.header_size), "Invalid torrent header: ");
|
2020-05-27 18:10:46 +00:00
|
|
|
|
2022-12-22 09:24:13 +00:00
|
|
|
Torrent torrent(info, std::move(header), std::move(tree), std::move(chunks), root_dir_);
|
2020-05-27 18:10:46 +00:00
|
|
|
|
|
|
|
return std::move(torrent);
|
|
|
|
}
|
|
|
|
} // namespace ton
|