external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

TON Storage utilities (#564)

Browse source 
* Rename chunk to piece in MerkleTree for consistency

* Refactor PeerManager

* Make PeerState thread-safe

* Download torrent by hash

* First version of storage daemon

* Download torrents partially

* Improve storing and loading torrent state in DB

* Rewrite MerkleTree

* "Remove torrent" in storage daemon

* Process errors, fix bugs in storage

* Move TonlibClientWrapper from rldp-http-proxy to tonlib

* Initial version of storage provider

* Move interaction with contracts to smc-util

* Improve TonlibClientWrapper interface

* Various improvements in storage provider

* Fix TorrentCreator.cpp

* Improve interface for partial download

* Client mode in storage-daemon

* Improve interface of storage-daemon-cli

* Fix calculating speed, show peers in storage-daemon

* Use permanent adnl id in storage daemon

* Fix sending large "storage.addUpdate" messages

* Improve printing torrents in cli

* Update tlo

* Fix RldpSender::on_ack

* Update storage provider

* Add "address" parameter to get-provider-params

* Allow client to close storage contract

* Limit torrent description

* Add more logs to storage provider

* smc.forget tonlib method

* Use smc.forget in storage daemon

* Optimize sending messages in smc-util.cpp

* Fix verbosity, remove excessive logs

* Json output in storage-daemon-cli

* Update storage provider contracts

* Fix rldp2 acks

* Change verbosity of logs in rldp2

* Update help and output of commands and in storage-daemon-cli

Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK 2022-12-22 12:24:13 +03:00 • committed by GitHub
parent 434dc487a4
commit 360ef54e6b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
75 changed files with 8872 additions and 1148 deletions
Download patch file Download diff file Expand all files Collapse all files

12
overlay/overlay-manager.cpp
View file

@ -90,12 +90,20 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho
}
void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) {
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope) {
create_public_overlay_ex(local_id, std::move(overlay_id), std::move(callback), std::move(rules), std::move(scope),
true);
}
void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope, bool announce_self) {
CHECK(!dht_node_.empty());
auto id = overlay_id.compute_short_id();
register_overlay(local_id, id,
Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id),
std::move(callback), std::move(rules), scope));
std::move(callback), std::move(rules), scope, announce_self));
}
void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,

3
overlay/overlay-manager.h
View file

@ -52,6 +52,9 @@ class OverlayManager : public Overlays {
void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) override;
void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope,
bool announce_self) override;
void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
OverlayPrivacyRules rules) override;

4
overlay/overlay-peers.cpp
View file

@ -171,7 +171,9 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result<td:
void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode node,
td::Promise<td::BufferSlice> promise) {
std::vector<tl_object_ptr<ton_api::overlay_node>> vec;
vec.emplace_back(node.tl());
if (announce_self_) {
vec.emplace_back(node.tl());
}
for (td::uint32 i = 0; i < nodes_to_send(); i++) {
auto P = get_random_peer(true);

13
overlay/overlay.cpp
View file

@ -37,10 +37,10 @@ td::actor::ActorOwn<Overlay> Overlay::create(td::actor::ActorId<keyring::Keyring
td::actor::ActorId<OverlayManager> manager,
td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
OverlayIdFull overlay_id, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope) {
OverlayPrivacyRules rules, td::string scope, bool announce_self) {
auto R = td::actor::create_actor<OverlayImpl>("overlay", keyring, adnl, manager, dht_node, local_id,
std::move(overlay_id), true, std::vector<adnl::AdnlNodeIdShort>(),
std::move(callback), std::move(rules), scope);
std::move(callback), std::move(rules), scope, announce_self);
return td::actor::ActorOwn<Overlay>(std::move(R));
}
@ -60,7 +60,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node,
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope)
OverlayPrivacyRules rules, td::string scope, bool announce_self)
: keyring_(keyring)
, adnl_(adnl)
, manager_(manager)
@ -70,7 +70,8 @@ OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor
, callback_(std::move(callback))
, public_(pub)
, rules_(std::move(rules))
, scope_(scope) {
, scope_(scope)
, announce_self_(announce_self) {
overlay_id_ = id_full_.compute_short_id();
VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private");
@ -328,6 +329,10 @@ void OverlayImpl::receive_dht_nodes(td::Result<dht::DhtValue> res, bool dummy) {
VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << res.move_as_error();
}
if (!announce_self_) {
return;
}
VLOG(OVERLAY_INFO) << this << ": adding self node to DHT overlay's nodes";
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), oid = print_id()](td::Result<OverlayNode> R) {
if (R.is_error()) {

2
overlay/overlay.h
View file

@ -42,7 +42,7 @@ class Overlay : public td::actor::Actor {
td::actor::ActorId<OverlayManager> manager,
td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
OverlayIdFull overlay_id, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope);
OverlayPrivacyRules rules, td::string scope, bool announce_self = true);
static td::actor::ActorOwn<Overlay> create(td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<OverlayManager> manager,

3
overlay/overlay.hpp
View file

@ -124,7 +124,7 @@ class OverlayImpl : public Overlay {
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node,
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }");
OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", bool announce_self = true);
void update_dht_node(td::actor::ActorId<dht::Dht> dht) override {
dht_node_ = dht;
}
@ -366,6 +366,7 @@ class OverlayImpl : public Overlay {
bool semi_public_ = false;
OverlayPrivacyRules rules_;
td::string scope_;
bool announce_self_ = true;
std::map<PublicKeyHash, std::shared_ptr<Certificate>> certs_;
class CachedEncryptor : public td::ListNode {

6
overlay/overlays.h
View file

@ -193,7 +193,11 @@ class Overlays : public td::actor::Actor {
virtual void update_dht_node(td::actor::ActorId<dht::Dht> dht) = 0;
virtual void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) = 0;
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope) = 0;
virtual void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope, bool announce_self) = 0;
virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
OverlayPrivacyRules rules) = 0;