mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
TON Storage utilities (#564)
* Rename chunk to piece in MerkleTree for consistency * Refactor PeerManager * Make PeerState thread-safe * Download torrent by hash * First version of storage daemon * Download torrents partially * Improve storing and loading torrent state in DB * Rewrite MerkleTree * "Remove torrent" in storage daemon * Process errors, fix bugs in storage * Move TonlibClientWrapper from rldp-http-proxy to tonlib * Initial version of storage provider * Move interaction with contracts to smc-util * Improve TonlibClientWrapper interface * Various improvements in storage provider * Fix TorrentCreator.cpp * Improve interface for partial download * Client mode in storage-daemon * Improve interface of storage-daemon-cli * Fix calculating speed, show peers in storage-daemon * Use permanent adnl id in storage daemon * Fix sending large "storage.addUpdate" messages * Improve printing torrents in cli * Update tlo * Fix RldpSender::on_ack * Update storage provider * Add "address" parameter to get-provider-params * Allow client to close storage contract * Limit torrent description * Add more logs to storage provider * smc.forget tonlib method * Use smc.forget in storage daemon * Optimize sending messages in smc-util.cpp * Fix verbosity, remove excessive logs * Json output in storage-daemon-cli * Update storage provider contracts * Fix rldp2 acks * Change verbosity of logs in rldp2 * Update help and output of commands and in storage-daemon-cli Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK
GitHub
parent
434dc487a4
commit
360ef54e6b
75 changed files with 8872 additions and 1148 deletions
|
|
@ -1,5 +1,5 @@
|
|||
cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
|
||||
|
||||
add_executable(rldp-http-proxy rldp-http-proxy.cpp DNSResolver.h TonlibClient.h TonlibClient.cpp DNSResolver.cpp)
|
||||
add_executable(rldp-http-proxy rldp-http-proxy.cpp DNSResolver.h DNSResolver.cpp)
|
||||
target_include_directories(rldp-http-proxy PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
|
||||
target_link_libraries(rldp-http-proxy PRIVATE tonhttp rldp dht tonlib git)
|
||||
|
|
|
|||
|
|
@ -30,7 +30,8 @@
|
|||
static const double CACHE_TIMEOUT_HARD = 300.0;
|
||||
static const double CACHE_TIMEOUT_SOFT = 270.0;
|
||||
|
||||
DNSResolver::DNSResolver(td::actor::ActorId<TonlibClient> tonlib_client) : tonlib_client_(std::move(tonlib_client)) {
|
||||
DNSResolver::DNSResolver(td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client)
|
||||
: tonlib_client_(std::move(tonlib_client)) {
|
||||
}
|
||||
|
||||
void DNSResolver::start_up() {
|
||||
|
|
@ -39,14 +40,15 @@ void DNSResolver::start_up() {
|
|||
|
||||
void DNSResolver::sync() {
|
||||
auto obj = tonlib_api::make_object<tonlib_api::sync>();
|
||||
auto P = td::PromiseCreator::lambda([SelfId =
|
||||
actor_id(this)](td::Result<tonlib_api::object_ptr<tonlib_api::Object>> R) {
|
||||
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](
|
||||
td::Result<tonlib_api::object_ptr<tonlib_api::ton_blockIdExt>> R) {
|
||||
if (R.is_error()) {
|
||||
LOG(WARNING) << "Sync error: " << R.move_as_error();
|
||||
ton::delay_action([SelfId]() { td::actor::send_closure(SelfId, &DNSResolver::sync); }, td::Timestamp::in(5.0));
|
||||
}
|
||||
});
|
||||
td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P));
|
||||
td::actor::send_closure(tonlib_client_, &tonlib::TonlibClientWrapper::send_request<tonlib_api::sync>, std::move(obj),
|
||||
std::move(P));
|
||||
}
|
||||
|
||||
void DNSResolver::resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdShort> promise) {
|
||||
|
|
@ -66,18 +68,13 @@ void DNSResolver::resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdSho
|
|||
td::Bits256 category = td::sha256_bits256(td::Slice("site", 4));
|
||||
auto obj = tonlib_api::make_object<tonlib_api::dns_resolve>(nullptr, host, category, 16);
|
||||
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise), host = std::move(host)](
|
||||
td::Result<tonlib_api::object_ptr<tonlib_api::Object>> R) mutable {
|
||||
td::Result<tonlib_api::object_ptr<tonlib_api::dns_resolved>> R) mutable {
|
||||
if (R.is_error()) {
|
||||
if (promise) {
|
||||
promise.set_result(R.move_as_error());
|
||||
}
|
||||
} else {
|
||||
auto v = R.move_as_ok();
|
||||
auto obj = dynamic_cast<tonlib_api::dns_resolved *>(v.get());
|
||||
if (obj == nullptr) {
|
||||
promise.set_result(td::Status::Error("invalid response from tonlib"));
|
||||
return;
|
||||
}
|
||||
auto obj = R.move_as_ok();
|
||||
ton::adnl::AdnlNodeIdShort id;
|
||||
td::uint32 cnt = 0;
|
||||
for (auto &e : obj->entries_) {
|
||||
|
|
@ -106,7 +103,8 @@ void DNSResolver::resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdSho
|
|||
}
|
||||
}
|
||||
});
|
||||
td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P));
|
||||
td::actor::send_closure(tonlib_client_, &tonlib::TonlibClientWrapper::send_request<tonlib_api::dns_resolve>,
|
||||
std::move(obj), std::move(P));
|
||||
}
|
||||
|
||||
void DNSResolver::save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id) {
|
||||
|
|
|
|||
|
|
@ -25,13 +25,13 @@
|
|||
*/
|
||||
#pragma once
|
||||
#include "td/actor/actor.h"
|
||||
#include "TonlibClient.h"
|
||||
#include "tonlib/tonlib/TonlibClientWrapper.h"
|
||||
#include "adnl/adnl.h"
|
||||
#include "td/actor/PromiseFuture.h"
|
||||
|
||||
class DNSResolver : public td::actor::Actor {
|
||||
public:
|
||||
explicit DNSResolver(td::actor::ActorId<TonlibClient> tonlib_client);
|
||||
explicit DNSResolver(td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client);
|
||||
|
||||
void start_up() override;
|
||||
void resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdShort> promise);
|
||||
|
|
@ -40,7 +40,7 @@ class DNSResolver : public td::actor::Actor {
|
|||
void sync();
|
||||
void save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id);
|
||||
|
||||
td::actor::ActorId<TonlibClient> tonlib_client_;
|
||||
td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client_;
|
||||
|
||||
struct CacheEntry {
|
||||
ton::adnl::AdnlNodeIdShort id_;
|
||||
|
|
|
|||
|
|
@ -1,72 +0,0 @@
|
|||
/*
|
||||
This file is part of TON Blockchain source code.
|
||||
|
||||
TON Blockchain is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU General Public License
|
||||
as published by the Free Software Foundation; either version 2
|
||||
of the License, or (at your option) any later version.
|
||||
|
||||
TON Blockchain is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with TON Blockchain. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
In addition, as a special exception, the copyright holders give permission
|
||||
to link the code of portions of this program with the OpenSSL library.
|
||||
You must obey the GNU General Public License in all respects for all
|
||||
of the code used other than OpenSSL. If you modify file(s) with this
|
||||
exception, you may extend this exception to your version of the file(s),
|
||||
but you are not obligated to do so. If you do not wish to do so, delete this
|
||||
exception statement from your version. If you delete this exception statement
|
||||
from all source files in the program, then also delete it here.
|
||||
*/
|
||||
#include "TonlibClient.h"
|
||||
|
||||
TonlibClient::TonlibClient(ton::tl_object_ptr<tonlib_api::options> options) : options_(std::move(options)) {
|
||||
}
|
||||
|
||||
void TonlibClient::start_up() {
|
||||
class Cb : public tonlib::TonlibCallback {
|
||||
public:
|
||||
explicit Cb(td::actor::ActorId<TonlibClient> self_id) : self_id_(self_id) {
|
||||
}
|
||||
void on_result(std::uint64_t id, tonlib_api::object_ptr<tonlib_api::Object> result) override {
|
||||
td::actor::send_closure(self_id_, &TonlibClient::receive_request_result, id, std::move(result));
|
||||
}
|
||||
void on_error(std::uint64_t id, tonlib_api::object_ptr<tonlib_api::error> error) override {
|
||||
td::actor::send_closure(self_id_, &TonlibClient::receive_request_result, id,
|
||||
td::Status::Error(error->code_, std::move(error->message_)));
|
||||
}
|
||||
|
||||
private:
|
||||
td::actor::ActorId<TonlibClient> self_id_;
|
||||
};
|
||||
|
||||
tonlib_client_ = td::actor::create_actor<tonlib::TonlibClient>("tonlibclient", td::make_unique<Cb>(actor_id(this)));
|
||||
auto init = tonlib_api::make_object<tonlib_api::init>(std::move(options_));
|
||||
auto P = td::PromiseCreator::lambda([](td::Result<tonlib_api::object_ptr<tonlib_api::Object>> R) mutable {
|
||||
R.ensure();
|
||||
});
|
||||
send_request(std::move(init), std::move(P));
|
||||
}
|
||||
|
||||
void TonlibClient::send_request(tonlib_api::object_ptr<tonlib_api::Function> obj,
|
||||
td::Promise<tonlib_api::object_ptr<tonlib_api::Object>> promise) {
|
||||
auto id = next_request_id_++;
|
||||
CHECK(requests_.emplace(id, std::move(promise)).second);
|
||||
td::actor::send_closure(tonlib_client_, &tonlib::TonlibClient::request, id, std::move(obj));
|
||||
}
|
||||
|
||||
void TonlibClient::receive_request_result(td::uint64 id, td::Result<tonlib_api::object_ptr<tonlib_api::Object>> R) {
|
||||
if (id == 0) {
|
||||
return;
|
||||
}
|
||||
auto it = requests_.find(id);
|
||||
CHECK(it != requests_.end());
|
||||
auto promise = std::move(it->second);
|
||||
requests_.erase(it);
|
||||
promise.set_result(std::move(R));
|
||||
}
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
/*
|
||||
This file is part of TON Blockchain source code.
|
||||
|
||||
TON Blockchain is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU General Public License
|
||||
as published by the Free Software Foundation; either version 2
|
||||
of the License, or (at your option) any later version.
|
||||
|
||||
TON Blockchain is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with TON Blockchain. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
In addition, as a special exception, the copyright holders give permission
|
||||
to link the code of portions of this program with the OpenSSL library.
|
||||
You must obey the GNU General Public License in all respects for all
|
||||
of the code used other than OpenSSL. If you modify file(s) with this
|
||||
exception, you may extend this exception to your version of the file(s),
|
||||
but you are not obligated to do so. If you do not wish to do so, delete this
|
||||
exception statement from your version. If you delete this exception statement
|
||||
from all source files in the program, then also delete it here.
|
||||
*/
|
||||
#pragma once
|
||||
#include "td/actor/actor.h"
|
||||
#include "auto/tl/tonlib_api.hpp"
|
||||
#include "tonlib/tonlib/TonlibClient.h"
|
||||
|
||||
class TonlibClient : public td::actor::Actor {
|
||||
public:
|
||||
explicit TonlibClient(ton::tl_object_ptr<tonlib_api::options> options);
|
||||
|
||||
void start_up() override;
|
||||
|
||||
void send_request(tonlib_api::object_ptr<tonlib_api::Function> obj,
|
||||
td::Promise<tonlib_api::object_ptr<tonlib_api::Object>> promise);
|
||||
|
||||
private:
|
||||
void receive_request_result(td::uint64 id, td::Result<tonlib_api::object_ptr<tonlib_api::Object>> R);
|
||||
|
||||
ton::tl_object_ptr<tonlib_api::options> options_;
|
||||
td::actor::ActorOwn<tonlib::TonlibClient> tonlib_client_;
|
||||
std::map<td::uint64, td::Promise<tonlib_api::object_ptr<tonlib_api::Object>>> requests_;
|
||||
td::uint64 next_request_id_{1};
|
||||
};
|
||||
|
|
@ -54,7 +54,7 @@
|
|||
#include "td/utils/BufferedFd.h"
|
||||
#include "common/delay.h"
|
||||
|
||||
#include "TonlibClient.h"
|
||||
#include "tonlib/tonlib/TonlibClientWrapper.h"
|
||||
#include "DNSResolver.h"
|
||||
|
||||
#if TD_DARWIN || TD_LINUX
|
||||
|
|
@ -919,7 +919,7 @@ class RldpHttpProxy : public td::actor::Actor {
|
|||
auto tonlib_options = tonlib_api::make_object<tonlib_api::options>(
|
||||
tonlib_api::make_object<tonlib_api::config>(conf_dataR.move_as_ok().as_slice().str(), "", false, false),
|
||||
tonlib_api::make_object<tonlib_api::keyStoreTypeInMemory>());
|
||||
tonlib_client_ = td::actor::create_actor<TonlibClient>("tonlibclient", std::move(tonlib_options));
|
||||
tonlib_client_ = td::actor::create_actor<tonlib::TonlibClientWrapper>("tonlibclient", std::move(tonlib_options));
|
||||
dns_resolver_ = td::actor::create_actor<DNSResolver>("dnsresolver", tonlib_client_.get());
|
||||
}
|
||||
|
||||
|
|
@ -1315,7 +1315,7 @@ class RldpHttpProxy : public td::actor::Actor {
|
|||
std::string db_root_ = ".";
|
||||
bool proxy_all_ = false;
|
||||
|
||||
td::actor::ActorOwn<TonlibClient> tonlib_client_;
|
||||
td::actor::ActorOwn<tonlib::TonlibClientWrapper> tonlib_client_;
|
||||
td::actor::ActorOwn<DNSResolver> dns_resolver_;
|
||||
|
||||
std::map<td::Bits256,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue