external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Improve DHT store/load, pinging overlay peers (#840)

Browse source 
* Improve DHT store/load, pinging overlay peers

* Fix speed limits in storage

* Use keyStoreTypeDirectory in rldp-http-proxy and storage-daemon

Mainly for caching synced block in tonlib.

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK 2023-12-28 09:43:10 +03:00 • committed by GitHub
parent c8918f0c02
commit 550c28d7db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 162 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

6
overlay/overlay-manager.cpp
View file

@ -93,17 +93,17 @@ void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, Overl
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope) {
create_public_overlay_ex(local_id, std::move(overlay_id), std::move(callback), std::move(rules), std::move(scope),
true);
{});
}
void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope, bool announce_self) {
td::string scope, OverlayOptions opts) {
CHECK(!dht_node_.empty());
auto id = overlay_id.compute_short_id();
register_overlay(local_id, id,
Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id),
std::move(callback), std::move(rules), scope, announce_self));
std::move(callback), std::move(rules), scope, std::move(opts)));
}
void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,

2
overlay/overlay-manager.h
View file

@ -54,7 +54,7 @@ class OverlayManager : public Overlays {
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) override;
void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope,
bool announce_self) override;
OverlayOptions opts) override;
void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
OverlayPrivacyRules rules) override;

23
overlay/overlay.cpp
View file

@ -37,10 +37,10 @@ td::actor::ActorOwn<Overlay> Overlay::create(td::actor::ActorId<keyring::Keyring
td::actor::ActorId<OverlayManager> manager,
td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
OverlayIdFull overlay_id, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope, bool announce_self) {
OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) {
auto R = td::actor::create_actor<OverlayImpl>("overlay", keyring, adnl, manager, dht_node, local_id,
std::move(overlay_id), true, std::vector<adnl::AdnlNodeIdShort>(),
std::move(callback), std::move(rules), scope, announce_self);
std::move(callback), std::move(rules), scope, opts);
return td::actor::ActorOwn<Overlay>(std::move(R));
}
@ -60,7 +60,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node,
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope, bool announce_self)
OverlayPrivacyRules rules, td::string scope, OverlayOptions opts)
: keyring_(keyring)
, adnl_(adnl)
, manager_(manager)
@ -71,7 +71,8 @@ OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor
, public_(pub)
, rules_(std::move(rules))
, scope_(scope)
, announce_self_(announce_self) {
, announce_self_(opts.announce_self_)
, frequent_dht_lookup_(opts.frequent_dht_lookup_) {
overlay_id_ = id_full_.compute_short_id();
VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private");
@ -279,13 +280,13 @@ void OverlayImpl::alarm() {
send_random_peers(P->get_id(), {});
}
}
if (next_dht_query_.is_in_past()) {
if (next_dht_query_ && next_dht_query_.is_in_past()) {
next_dht_query_ = td::Timestamp::never();
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<dht::DhtValue> res) {
td::actor::send_closure(SelfId, &OverlayImpl::receive_dht_nodes, std::move(res), true);
});
td::actor::send_closure(dht_node_, &dht::Dht::get_value, dht::DhtKey{overlay_id_.pubkey_hash(), "nodes", 0},
std::move(P));
next_dht_query_ = td::Timestamp::in(td::Random::fast(60.0, 100.0));
}
if (update_db_at_.is_in_past()) {
if (peers_.size() > 0) {
@ -333,7 +334,13 @@ void OverlayImpl::receive_dht_nodes(td::Result<dht::DhtValue> res, bool dummy) {
VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << res.move_as_error();
}
if (!(next_dht_store_query_ && next_dht_store_query_.is_in_past())) {
finish_dht_query();
return;
}
next_dht_store_query_ = td::Timestamp::never();
if (!announce_self_) {
finish_dht_query();
return;
}
@ -341,6 +348,7 @@ void OverlayImpl::receive_dht_nodes(td::Result<dht::DhtValue> res, bool dummy) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), oid = print_id()](td::Result<OverlayNode> R) {
if (R.is_error()) {
LOG(ERROR) << oid << "cannot get self node";
td::actor::send_closure(SelfId, &OverlayImpl::finish_dht_query);
return;
}
td::actor::send_closure(SelfId, &OverlayImpl::update_dht_nodes, R.move_as_ok());
@ -365,10 +373,11 @@ void OverlayImpl::update_dht_nodes(OverlayNode node) {
static_cast<td::uint32>(td::Clocks::system() + 3600), td::BufferSlice()};
value.check().ensure();
auto P = td::PromiseCreator::lambda([oid = print_id()](td::Result<td::Unit> res) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), oid = print_id()](td::Result<td::Unit> res) {
if (res.is_error()) {
VLOG(OVERLAY_NOTICE) << oid << ": error storing to DHT: " << res.move_as_error();
}
td::actor::send_closure(SelfId, &OverlayImpl::finish_dht_query);
});
td::actor::send_closure(dht_node_, &dht::Dht::set_value, std::move(value), std::move(P));

2
overlay/overlay.h
View file

@ -42,7 +42,7 @@ class Overlay : public td::actor::Actor {
td::actor::ActorId<OverlayManager> manager,
td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
OverlayIdFull overlay_id, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope, bool announce_self = true);
OverlayPrivacyRules rules, td::string scope, OverlayOptions opts = {});
static td::actor::ActorOwn<Overlay> create(td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<OverlayManager> manager,

24
overlay/overlay.hpp
View file

@ -82,12 +82,17 @@ class OverlayPeer {
void on_ping_result(bool success) {
if (success) {
missed_pings_ = 0;
last_ping_at_ = td::Timestamp::now();
is_alive_ = true;
} else {
++missed_pings_;
if (missed_pings_ >= 3 && last_ping_at_.is_in_past(td::Timestamp::in(-15.0))) {
is_alive_ = false;
}
}
}
bool is_alive() const {
return missed_pings_ < 3;
return is_alive_;
}
td::uint32 throughput_out_bytes = 0;
@ -116,6 +121,8 @@ class OverlayPeer {
bool is_neighbour_ = false;
size_t missed_pings_ = 0;
bool is_alive_ = true;
td::Timestamp last_ping_at_ = td::Timestamp::now();
};
class OverlayImpl : public Overlay {
@ -124,7 +131,7 @@ class OverlayImpl : public Overlay {
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node,
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", bool announce_self = true);
OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", OverlayOptions opts = {});
void update_dht_node(td::actor::ActorId<dht::Dht> dht) override {
dht_node_ = dht;
}
@ -295,6 +302,17 @@ class OverlayImpl : public Overlay {
void del_peer(adnl::AdnlNodeIdShort id);
OverlayPeer *get_random_peer(bool only_alive = false);
void finish_dht_query() {
if (!next_dht_store_query_) {
next_dht_store_query_ = td::Timestamp::in(td::Random::fast(60.0, 100.0));
}
if (frequent_dht_lookup_ && peers_.size() == bad_peers_.size()) {
next_dht_query_ = td::Timestamp::in(td::Random::fast(6.0, 10.0));
} else {
next_dht_query_ = next_dht_store_query_;
}
}
td::actor::ActorId<keyring::Keyring> keyring_;
td::actor::ActorId<adnl::Adnl> adnl_;
td::actor::ActorId<OverlayManager> manager_;
@ -305,6 +323,7 @@ class OverlayImpl : public Overlay {
td::DecTree<adnl::AdnlNodeIdShort, OverlayPeer> peers_;
td::Timestamp next_dht_query_ = td::Timestamp::in(1.0);
td::Timestamp next_dht_store_query_ = td::Timestamp::in(1.0);
td::Timestamp update_db_at_;
td::Timestamp update_throughput_at_;
td::Timestamp last_throughput_update_;
@ -367,6 +386,7 @@ class OverlayImpl : public Overlay {
OverlayPrivacyRules rules_;
td::string scope_;
bool announce_self_ = true;
bool frequent_dht_lookup_ = false;
std::map<PublicKeyHash, std::shared_ptr<Certificate>> certs_;
class CachedEncryptor : public td::ListNode {

7
overlay/overlays.h
View file

@ -158,6 +158,11 @@ class Certificate {
td::SharedSlice signature_;
};
struct OverlayOptions {
bool announce_self_ = true;
bool frequent_dht_lookup_ = false;
};
class Overlays : public td::actor::Actor {
public:
class Callback {
@ -197,7 +202,7 @@ class Overlays : public td::actor::Actor {
td::string scope) = 0;
virtual void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope, bool announce_self) = 0;
td::string scope, OverlayOptions opts) = 0;
virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
OverlayPrivacyRules rules) = 0;