mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
TON Storage utilities (#564)
* Rename chunk to piece in MerkleTree for consistency * Refactor PeerManager * Make PeerState thread-safe * Download torrent by hash * First version of storage daemon * Download torrents partially * Improve storing and loading torrent state in DB * Rewrite MerkleTree * "Remove torrent" in storage daemon * Process errors, fix bugs in storage * Move TonlibClientWrapper from rldp-http-proxy to tonlib * Initial version of storage provider * Move interaction with contracts to smc-util * Improve TonlibClientWrapper interface * Various improvements in storage provider * Fix TorrentCreator.cpp * Improve interface for partial download * Client mode in storage-daemon * Improve interface of storage-daemon-cli * Fix calculating speed, show peers in storage-daemon * Use permanent adnl id in storage daemon * Fix sending large "storage.addUpdate" messages * Improve printing torrents in cli * Update tlo * Fix RldpSender::on_ack * Update storage provider * Add "address" parameter to get-provider-params * Allow client to close storage contract * Limit torrent description * Add more logs to storage provider * smc.forget tonlib method * Use smc.forget in storage daemon * Optimize sending messages in smc-util.cpp * Fix verbosity, remove excessive logs * Json output in storage-daemon-cli * Update storage provider contracts * Fix rldp2 acks * Change verbosity of logs in rldp2 * Update help and output of commands and in storage-daemon-cli Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK
GitHub
parent
434dc487a4
commit
360ef54e6b
75 changed files with 8872 additions and 1148 deletions
|
|
@ -25,42 +25,87 @@
|
|||
#include "Torrent.h"
|
||||
|
||||
#include "td/utils/Random.h"
|
||||
#include "td/utils/Variant.h"
|
||||
|
||||
#include <map>
|
||||
#include "db.h"
|
||||
|
||||
namespace ton {
|
||||
class NodeActor : public td::actor::Actor {
|
||||
public:
|
||||
class NodeCallback {
|
||||
public:
|
||||
virtual ~NodeCallback() = default;
|
||||
virtual td::actor::ActorOwn<PeerActor> create_peer(PeerId self_id, PeerId peer_id,
|
||||
std::shared_ptr<PeerState> state) = 0;
|
||||
virtual void get_peers(PeerId src, td::Promise<std::vector<PeerId>> peers) = 0;
|
||||
virtual void register_self(td::actor::ActorId<ton::NodeActor> self) = 0;
|
||||
virtual void get_peer_info(PeerId src, PeerId peer, td::Promise<std::pair<td::Bits256, std::string>> promise) {
|
||||
promise.set_error(td::Status::Error("Not implemented"));
|
||||
}
|
||||
};
|
||||
|
||||
class Callback {
|
||||
public:
|
||||
virtual ~Callback() {
|
||||
}
|
||||
virtual td::actor::ActorOwn<PeerActor> create_peer(PeerId self_id, PeerId peer_id,
|
||||
td::SharedState<PeerState> state) = 0;
|
||||
virtual void get_peers(td::Promise<std::vector<PeerId>> peers) = 0;
|
||||
virtual void register_self(td::actor::ActorId<ton::NodeActor> self) = 0;
|
||||
|
||||
//TODO: proper callbacks
|
||||
virtual ~Callback() = default;
|
||||
virtual void on_completed() = 0;
|
||||
virtual void on_closed(ton::Torrent torrent) = 0;
|
||||
};
|
||||
|
||||
NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback, bool should_download = true);
|
||||
struct PendingSetFilePriority {
|
||||
struct All {};
|
||||
td::Variant<All, size_t, std::string> file;
|
||||
td::uint8 priority;
|
||||
};
|
||||
struct DbInitialData {
|
||||
std::vector<PendingSetFilePriority> priorities;
|
||||
std::set<td::uint64> pieces_in_db;
|
||||
};
|
||||
|
||||
NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback,
|
||||
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download = true);
|
||||
NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback,
|
||||
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download,
|
||||
DbInitialData db_initial_data);
|
||||
void start_peer(PeerId peer_id, td::Promise<td::actor::ActorId<PeerActor>> promise);
|
||||
|
||||
ton::Torrent *with_torrent() {
|
||||
return &torrent_;
|
||||
struct NodeState {
|
||||
Torrent &torrent;
|
||||
bool active_download;
|
||||
double download_speed;
|
||||
double upload_speed;
|
||||
const std::vector<td::uint8> &file_priority;
|
||||
};
|
||||
void with_torrent(td::Promise<NodeState> promise) {
|
||||
promise.set_value(
|
||||
NodeState{torrent_, should_download_, download_speed_.speed(), upload_speed_.speed(), file_priority_});
|
||||
}
|
||||
std::string get_stats_str();
|
||||
|
||||
void set_file_priority(size_t i, td::uint8 priority);
|
||||
void set_should_download(bool should_download);
|
||||
|
||||
void set_all_files_priority(td::uint8 priority, td::Promise<bool> promise);
|
||||
void set_file_priority_by_idx(size_t i, td::uint8 priority, td::Promise<bool> promise);
|
||||
void set_file_priority_by_name(std::string name, td::uint8 priority, td::Promise<bool> promise);
|
||||
|
||||
void load_from(td::optional<TorrentMeta> meta, std::string files_path, td::Promise<td::Unit> promise);
|
||||
|
||||
void wait_for_completion(td::Promise<td::Unit> promise);
|
||||
void get_peers_info(td::Promise<tl_object_ptr<ton_api::storage_daemon_peerList>> promise);
|
||||
|
||||
static void load_from_db(std::shared_ptr<db::DbType> db, td::Bits256 hash, td::unique_ptr<Callback> callback,
|
||||
td::unique_ptr<NodeCallback> node_callback,
|
||||
td::Promise<td::actor::ActorOwn<NodeActor>> promise);
|
||||
static void cleanup_db(std::shared_ptr<db::DbType> db, td::Bits256 hash, td::Promise<td::Unit> promise);
|
||||
|
||||
private:
|
||||
PeerId self_id_;
|
||||
ton::Torrent torrent_;
|
||||
std::shared_ptr<td::BufferSlice> torrent_info_str_;
|
||||
std::vector<td::uint8> file_priority_;
|
||||
td::unique_ptr<Callback> callback_;
|
||||
td::unique_ptr<NodeCallback> node_callback_;
|
||||
std::shared_ptr<db::DbType> db_;
|
||||
bool should_download_{false};
|
||||
|
||||
class Notifier : public td::actor::Actor {
|
||||
|
|
@ -80,24 +125,13 @@ class NodeActor : public td::actor::Actor {
|
|||
struct Peer {
|
||||
td::actor::ActorOwn<PeerActor> actor;
|
||||
td::actor::ActorOwn<Notifier> notifier;
|
||||
td::SharedState<PeerState> state;
|
||||
std::shared_ptr<PeerState> state;
|
||||
PartsHelper::PeerToken peer_token;
|
||||
LoadSpeed download_speed, upload_speed;
|
||||
};
|
||||
|
||||
std::map<PeerId, Peer> peers_;
|
||||
|
||||
struct QueryId {
|
||||
PeerId peer;
|
||||
PartId part;
|
||||
|
||||
auto key() const {
|
||||
return std::tie(peer, part);
|
||||
}
|
||||
bool operator<(const QueryId &other) const {
|
||||
return key() < other.key();
|
||||
}
|
||||
};
|
||||
|
||||
struct PartsSet {
|
||||
struct Info {
|
||||
td::optional<PeerId> query_to_peer;
|
||||
|
|
@ -110,7 +144,7 @@ class NodeActor : public td::actor::Actor {
|
|||
PartsSet parts_;
|
||||
PartsHelper parts_helper_;
|
||||
std::vector<PartId> ready_parts_;
|
||||
LoadSpeed download_;
|
||||
LoadSpeed download_speed_, upload_speed_;
|
||||
|
||||
td::Timestamp next_get_peers_at_;
|
||||
bool has_get_peers_{false};
|
||||
|
|
@ -118,9 +152,22 @@ class NodeActor : public td::actor::Actor {
|
|||
static constexpr double GET_PEER_EACH = 5;
|
||||
|
||||
bool is_completed_{false};
|
||||
std::vector<td::Promise<td::Unit>> wait_for_completion_;
|
||||
|
||||
td::Timestamp will_upload_at_;
|
||||
|
||||
std::vector<PendingSetFilePriority> pending_set_file_priority_;
|
||||
bool header_ready_ = false;
|
||||
std::map<std::string, size_t> file_name_to_idx_;
|
||||
std::set<td::uint64> pieces_in_db_;
|
||||
bool db_store_priorities_paused_ = false;
|
||||
td::int64 last_stored_meta_count_ = -1;
|
||||
td::Timestamp next_db_store_meta_at_ = td::Timestamp::now();
|
||||
|
||||
void init_torrent();
|
||||
void init_torrent_header();
|
||||
void recheck_parts(Torrent::PartsRange range);
|
||||
|
||||
void on_signal_from_peer(PeerId peer_id);
|
||||
|
||||
void start_up() override;
|
||||
|
|
@ -134,13 +181,23 @@ class NodeActor : public td::actor::Actor {
|
|||
static constexpr size_t MAX_TOTAL_QUERIES = 20;
|
||||
static constexpr size_t MAX_PEER_TOTAL_QUERIES = 5;
|
||||
void loop_queries();
|
||||
bool try_send_query();
|
||||
bool try_send_part(PartId part_id);
|
||||
void loop_get_peers();
|
||||
void got_peers(td::Result<std::vector<PeerId>> r_peers);
|
||||
void loop_peer(const PeerId &peer_id, Peer &peer);
|
||||
void on_part_ready(PartId part_id);
|
||||
|
||||
void loop_will_upload();
|
||||
|
||||
void got_torrent_info_str(td::BufferSlice data);
|
||||
|
||||
void update_pieces_in_db(td::uint64 begin, td::uint64 end);
|
||||
|
||||
void db_store_torrent();
|
||||
void db_store_priorities();
|
||||
void db_store_torrent_meta();
|
||||
void after_db_store_torrent_meta(td::Result<td::int64> R);
|
||||
void db_store_piece(td::uint64 i, std::string s);
|
||||
void db_erase_piece(td::uint64 i);
|
||||
void db_update_pieces_list();
|
||||
};
|
||||
} // namespace ton
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue