mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
Accelerator, part 1 (#1119)
This commit contains some parts of https://github.com/ton-blockchain/ton/tree/accelerator This is auxiliary code that mostly does not change node behavior. 1) Semiprivate overlays and other improvements in overlays code 2) Rename actual_min_split -> monitor_min_split, fix building shard overlays 3) Loading block candidates by block id from DB, fix accept_block after validator restart 4) Cells: ProofStorageStat and changes in CellUsageTree 5) Remove some unused code, other minor changes
This commit is contained in:
SpyCheese
GitHub
parent
9a10f79fba
commit
908415d00b
66 changed files with 2221 additions and 638 deletions
|
|
@ -18,11 +18,15 @@
|
|||
*/
|
||||
#pragma once
|
||||
|
||||
#include <any>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <unordered_set>
|
||||
#include <queue>
|
||||
|
||||
#include "adnl/adnl-node-id.hpp"
|
||||
#include "overlay.h"
|
||||
#include "overlay-manager.h"
|
||||
#include "overlay-fec.hpp"
|
||||
|
|
@ -32,6 +36,9 @@
|
|||
|
||||
#include "td/utils/DecTree.h"
|
||||
#include "td/utils/List.h"
|
||||
#include "td/utils/Status.h"
|
||||
#include "td/utils/Time.h"
|
||||
#include "td/utils/buffer.h"
|
||||
#include "td/utils/overloaded.h"
|
||||
#include "fec/fec.h"
|
||||
|
||||
|
|
@ -40,6 +47,8 @@
|
|||
|
||||
#include "auto/tl/ton_api.h"
|
||||
#include "auto/tl/ton_api.hpp"
|
||||
#include "td/utils/port/signals.h"
|
||||
#include "tl-utils/common-utils.hpp"
|
||||
|
||||
namespace ton {
|
||||
|
||||
|
|
@ -58,15 +67,17 @@ class OverlayPeer {
|
|||
adnl::AdnlNodeIdFull get_full_id() const {
|
||||
return node_.adnl_id_full();
|
||||
}
|
||||
OverlayNode get() const {
|
||||
return node_.clone();
|
||||
const OverlayNode *get_node() const {
|
||||
return &node_;
|
||||
}
|
||||
void update(OverlayNode node) {
|
||||
CHECK(get_id() == node.adnl_id_short());
|
||||
if (node.version() > node_.version()) {
|
||||
node_ = std::move(node);
|
||||
}
|
||||
node_.update(std::move(node));
|
||||
}
|
||||
void update_certificate(OverlayMemberCertificate cert) {
|
||||
node_.update_certificate(std::move(cert));
|
||||
}
|
||||
|
||||
OverlayPeer(OverlayNode node) : node_(std::move(node)) {
|
||||
id_ = node_.adnl_id_short();
|
||||
}
|
||||
|
|
@ -95,24 +106,44 @@ class OverlayPeer {
|
|||
return is_alive_;
|
||||
}
|
||||
|
||||
bool is_permanent_member() const {
|
||||
return is_permanent_member_;
|
||||
}
|
||||
|
||||
void set_permanent(bool value) {
|
||||
is_permanent_member_ = value;
|
||||
}
|
||||
|
||||
void clear_certificate() {
|
||||
node_.clear_certificate();
|
||||
}
|
||||
|
||||
auto certificate() const {
|
||||
return node_.certificate();
|
||||
}
|
||||
|
||||
bool has_full_id() const {
|
||||
return node_.has_full_id();
|
||||
}
|
||||
|
||||
td::uint32 throughput_out_bytes = 0;
|
||||
td::uint32 throughput_in_bytes = 0;
|
||||
|
||||
|
||||
td::uint32 throughput_out_packets = 0;
|
||||
td::uint32 throughput_in_packets = 0;
|
||||
|
||||
|
||||
td::uint32 throughput_out_bytes_ctr = 0;
|
||||
td::uint32 throughput_in_bytes_ctr = 0;
|
||||
|
||||
|
||||
td::uint32 throughput_out_packets_ctr = 0;
|
||||
td::uint32 throughput_in_packets_ctr = 0;
|
||||
|
||||
|
||||
td::uint32 broadcast_errors = 0;
|
||||
td::uint32 fec_broadcast_errors = 0;
|
||||
|
||||
|
||||
td::Timestamp last_in_query_at = td::Timestamp::now();
|
||||
td::Timestamp last_out_query_at = td::Timestamp::now();
|
||||
|
||||
|
||||
td::string ip_addr_str = "undefined";
|
||||
|
||||
private:
|
||||
|
|
@ -122,6 +153,7 @@ class OverlayPeer {
|
|||
bool is_neighbour_ = false;
|
||||
size_t missed_pings_ = 0;
|
||||
bool is_alive_ = true;
|
||||
bool is_permanent_member_ = false;
|
||||
td::Timestamp last_ping_at_ = td::Timestamp::now();
|
||||
};
|
||||
|
||||
|
|
@ -129,19 +161,23 @@ class OverlayImpl : public Overlay {
|
|||
public:
|
||||
OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
|
||||
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node,
|
||||
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub,
|
||||
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
|
||||
OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", OverlayOptions opts = {});
|
||||
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, OverlayType overlay_type,
|
||||
std::vector<adnl::AdnlNodeIdShort> nodes, std::vector<PublicKeyHash> root_public_keys,
|
||||
OverlayMemberCertificate cert, std::unique_ptr<Overlays::Callback> callback, OverlayPrivacyRules rules,
|
||||
td::string scope = "{ \"type\": \"undefined\" }", OverlayOptions opts = {});
|
||||
void update_dht_node(td::actor::ActorId<dht::Dht> dht) override {
|
||||
dht_node_ = dht;
|
||||
}
|
||||
|
||||
void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) override;
|
||||
void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise<td::BufferSlice> promise) override;
|
||||
void receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr<ton_api::overlay_messageExtra> extra,
|
||||
td::BufferSlice data) override;
|
||||
void receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr<ton_api::overlay_messageExtra> extra,
|
||||
td::BufferSlice data, td::Promise<td::BufferSlice> promise) override;
|
||||
void send_message_to_neighbours(td::BufferSlice data) override;
|
||||
void send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) override;
|
||||
void send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) override;
|
||||
void receive_nodes_from_db(tl_object_ptr<ton_api::overlay_nodes> nodes) override;
|
||||
void receive_nodes_from_db_v2(tl_object_ptr<ton_api::overlay_nodesV2> nodes) override;
|
||||
|
||||
void get_self_node(td::Promise<OverlayNode> promise);
|
||||
|
||||
|
|
@ -149,8 +185,8 @@ class OverlayImpl : public Overlay {
|
|||
void start_up() override {
|
||||
update_throughput_at_ = td::Timestamp::in(50.0);
|
||||
last_throughput_update_ = td::Timestamp::now();
|
||||
|
||||
if (public_) {
|
||||
|
||||
if (overlay_type_ == OverlayType::Public) {
|
||||
update_db_at_ = td::Timestamp::in(60.0);
|
||||
}
|
||||
alarm_timestamp() = td::Timestamp::in(1);
|
||||
|
|
@ -158,13 +194,17 @@ class OverlayImpl : public Overlay {
|
|||
|
||||
void on_ping_result(adnl::AdnlNodeIdShort peer, bool success);
|
||||
void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result<td::BufferSlice> R);
|
||||
void receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result<td::BufferSlice> R);
|
||||
void send_random_peers(adnl::AdnlNodeIdShort dst, td::Promise<td::BufferSlice> promise);
|
||||
void send_random_peers_v2(adnl::AdnlNodeIdShort dst, td::Promise<td::BufferSlice> promise);
|
||||
void send_random_peers_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise<td::BufferSlice> promise);
|
||||
void send_random_peers_v2_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise<td::BufferSlice> promise);
|
||||
void get_overlay_random_peers(td::uint32 max_peers, td::Promise<std::vector<adnl::AdnlNodeIdShort>> promise) override;
|
||||
void set_privacy_rules(OverlayPrivacyRules rules) override;
|
||||
void add_certificate(PublicKeyHash key, std::shared_ptr<Certificate> cert) override {
|
||||
certs_[key] = std::move(cert);
|
||||
}
|
||||
void update_member_certificate(OverlayMemberCertificate cert) override;
|
||||
|
||||
void receive_dht_nodes(dht::DhtValue v);
|
||||
void dht_lookup_finished(td::Status S);
|
||||
|
|
@ -188,6 +228,8 @@ class OverlayImpl : public Overlay {
|
|||
|
||||
td::Status check_date(td::uint32 date);
|
||||
BroadcastCheckResult check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec);
|
||||
BroadcastCheckResult check_source_eligible(const PublicKeyHash &source, const Certificate *cert, td::uint32 size,
|
||||
bool is_fec);
|
||||
td::Status check_delivered(BroadcastHash hash);
|
||||
|
||||
void broadcast_checked(Overlay::BroadcastHash hash, td::Result<td::Unit> R);
|
||||
|
|
@ -206,17 +248,7 @@ class OverlayImpl : public Overlay {
|
|||
void send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size,
|
||||
td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type,
|
||||
td::uint32 date);
|
||||
std::vector<adnl::AdnlNodeIdShort> get_neighbours(td::uint32 max_size = 0) const {
|
||||
if (max_size == 0 || max_size >= neighbours_.size()) {
|
||||
return neighbours_;
|
||||
} else {
|
||||
std::vector<adnl::AdnlNodeIdShort> vec;
|
||||
for (td::uint32 i = 0; i < max_size; i++) {
|
||||
vec.push_back(neighbours_[td::Random::fast(0, static_cast<td::int32>(neighbours_.size()) - 1)]);
|
||||
}
|
||||
return vec;
|
||||
}
|
||||
}
|
||||
std::vector<adnl::AdnlNodeIdShort> get_neighbours(td::uint32 max_size = 0) const;
|
||||
td::actor::ActorId<OverlayManager> overlay_manager() const {
|
||||
return manager_;
|
||||
}
|
||||
|
|
@ -236,39 +268,59 @@ class OverlayImpl : public Overlay {
|
|||
td::Result<Encryptor *> get_encryptor(PublicKey source);
|
||||
|
||||
void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlayStats>> promise) override;
|
||||
|
||||
void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override {
|
||||
auto out_peer = peers_.get(peer_id);
|
||||
if(out_peer) {
|
||||
out_peer->throughput_out_bytes_ctr += msg_size;
|
||||
out_peer->throughput_out_packets_ctr++;
|
||||
|
||||
if(is_query)
|
||||
{
|
||||
out_peer->last_out_query_at = td::Timestamp::now();
|
||||
}
|
||||
}
|
||||
|
||||
void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override;
|
||||
|
||||
void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override;
|
||||
|
||||
void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override;
|
||||
|
||||
void update_root_member_list(std::vector<adnl::AdnlNodeIdShort> nodes, std::vector<PublicKeyHash> root_public_keys,
|
||||
OverlayMemberCertificate cert) override;
|
||||
|
||||
bool is_valid_peer(const adnl::AdnlNodeIdShort &id, const ton_api::overlay_MemberCertificate *certificate);
|
||||
bool is_persistent_node(const adnl::AdnlNodeIdShort &id);
|
||||
|
||||
td::uint32 max_data_bcasts() const {
|
||||
return 100;
|
||||
}
|
||||
|
||||
void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override {
|
||||
auto in_peer = peers_.get(peer_id);
|
||||
if(in_peer) {
|
||||
in_peer->throughput_in_bytes_ctr += msg_size;
|
||||
in_peer->throughput_in_packets_ctr++;
|
||||
|
||||
if(is_query)
|
||||
{
|
||||
in_peer->last_in_query_at = td::Timestamp::now();
|
||||
}
|
||||
}
|
||||
td::uint32 max_bcasts() const {
|
||||
return 1000;
|
||||
}
|
||||
|
||||
void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override {
|
||||
auto fpeer = peers_.get(peer_id);
|
||||
if(fpeer) {
|
||||
fpeer->ip_addr_str = ip_str;
|
||||
}
|
||||
td::uint32 max_fec_bcasts() const {
|
||||
return 20;
|
||||
}
|
||||
td::uint32 max_sources() const {
|
||||
return 10;
|
||||
}
|
||||
td::uint32 max_encryptors() const {
|
||||
return 16;
|
||||
}
|
||||
|
||||
td::uint32 max_neighbours() const {
|
||||
return opts_.max_neighbours_;
|
||||
}
|
||||
|
||||
td::uint32 max_peers() const {
|
||||
return opts_.max_peers_;
|
||||
}
|
||||
|
||||
td::uint32 nodes_to_send() const {
|
||||
return opts_.nodes_to_send_;
|
||||
}
|
||||
|
||||
td::uint32 propagate_broadcast_to() const {
|
||||
return opts_.propagate_broadcast_to_;
|
||||
}
|
||||
|
||||
bool has_valid_membership_certificate();
|
||||
bool has_valid_broadcast_certificate(const PublicKeyHash &source, size_t size, bool is_fec);
|
||||
|
||||
void forget_peer(adnl::AdnlNodeIdShort peer_id) override {
|
||||
del_peer(peer_id);
|
||||
}
|
||||
|
||||
void wait_neighbours_not_empty(td::Promise<td::Unit> promise, int max_retries = 20);
|
||||
|
||||
private:
|
||||
template <class T>
|
||||
|
|
@ -278,6 +330,8 @@ class OverlayImpl : public Overlay {
|
|||
|
||||
void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query,
|
||||
td::Promise<td::BufferSlice> promise);
|
||||
void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query,
|
||||
td::Promise<td::BufferSlice> promise);
|
||||
void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query,
|
||||
td::Promise<td::BufferSlice> promise);
|
||||
void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query,
|
||||
|
|
@ -294,20 +348,28 @@ class OverlayImpl : public Overlay {
|
|||
td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr<ton_api::overlay_fec_completed> msg);
|
||||
td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr<ton_api::overlay_unicast> msg);
|
||||
|
||||
void do_add_peer(OverlayNode node);
|
||||
void add_peer_in_cont(OverlayNode node);
|
||||
void add_peer_in(OverlayNode node);
|
||||
td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate &cert);
|
||||
td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate *cert);
|
||||
td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, ton_api::overlay_MemberCertificate *cert);
|
||||
void add_peer(OverlayNode node);
|
||||
void add_peers(std::vector<OverlayNode> nodes);
|
||||
void add_peers(const tl_object_ptr<ton_api::overlay_nodes> &nodes);
|
||||
void add_peers(const tl_object_ptr<ton_api::overlay_nodesV2> &nodes);
|
||||
void del_some_peers();
|
||||
void del_peer(adnl::AdnlNodeIdShort id);
|
||||
void del_peer(const adnl::AdnlNodeIdShort &id);
|
||||
void del_from_neighbour_list(OverlayPeer *P);
|
||||
void del_from_neighbour_list(const adnl::AdnlNodeIdShort &id);
|
||||
void iterate_all_peers(std::function<void(const adnl::AdnlNodeIdShort &key, OverlayPeer &peer)> cb);
|
||||
OverlayPeer *get_random_peer(bool only_alive = false);
|
||||
bool is_root_public_key(const PublicKeyHash &key) const;
|
||||
bool has_good_peers() const;
|
||||
size_t neighbours_cnt() const;
|
||||
|
||||
void finish_dht_query() {
|
||||
if (!next_dht_store_query_) {
|
||||
next_dht_store_query_ = td::Timestamp::in(td::Random::fast(60.0, 100.0));
|
||||
}
|
||||
if (frequent_dht_lookup_ && peers_.size() == bad_peers_.size()) {
|
||||
if (frequent_dht_lookup_ && !has_good_peers()) {
|
||||
next_dht_query_ = td::Timestamp::in(td::Random::fast(6.0, 10.0));
|
||||
} else {
|
||||
next_dht_query_ = next_dht_store_query_;
|
||||
|
|
@ -322,14 +384,11 @@ class OverlayImpl : public Overlay {
|
|||
OverlayIdFull id_full_;
|
||||
OverlayIdShort overlay_id_;
|
||||
|
||||
td::DecTree<adnl::AdnlNodeIdShort, OverlayPeer> peers_;
|
||||
td::Timestamp next_dht_query_ = td::Timestamp::in(1.0);
|
||||
td::Timestamp next_dht_store_query_ = td::Timestamp::in(1.0);
|
||||
td::Timestamp update_db_at_;
|
||||
td::Timestamp update_throughput_at_;
|
||||
td::Timestamp last_throughput_update_;
|
||||
std::set<adnl::AdnlNodeIdShort> bad_peers_;
|
||||
adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero();
|
||||
|
||||
std::unique_ptr<Overlays::Callback> callback_;
|
||||
|
||||
|
|
@ -337,7 +396,6 @@ class OverlayImpl : public Overlay {
|
|||
std::map<BroadcastHash, std::unique_ptr<BroadcastFec>> fec_broadcasts_;
|
||||
std::set<BroadcastHash> delivered_broadcasts_;
|
||||
|
||||
std::vector<adnl::AdnlNodeIdShort> neighbours_;
|
||||
td::ListNode bcast_data_lru_;
|
||||
td::ListNode bcast_fec_lru_;
|
||||
std::queue<BroadcastHash> bcast_lru_;
|
||||
|
|
@ -346,33 +404,6 @@ class OverlayImpl : public Overlay {
|
|||
|
||||
void bcast_gc();
|
||||
|
||||
static td::uint32 max_data_bcasts() {
|
||||
return 100;
|
||||
}
|
||||
static td::uint32 max_bcasts() {
|
||||
return 1000;
|
||||
}
|
||||
static td::uint32 max_fec_bcasts() {
|
||||
return 20;
|
||||
}
|
||||
static td::uint32 max_sources() {
|
||||
return 10;
|
||||
}
|
||||
static td::uint32 max_neighbours() {
|
||||
return 5;
|
||||
}
|
||||
static td::uint32 max_encryptors() {
|
||||
return 16;
|
||||
}
|
||||
|
||||
static td::uint32 max_peers() {
|
||||
return 20;
|
||||
}
|
||||
|
||||
static td::uint32 nodes_to_send() {
|
||||
return 4;
|
||||
}
|
||||
|
||||
static BroadcastHash get_broadcast_hash(adnl::AdnlNodeIdShort &src, td::Bits256 &data_hash) {
|
||||
td::uint8 buf[64];
|
||||
td::MutableSlice m{buf, 64};
|
||||
|
|
@ -382,8 +413,7 @@ class OverlayImpl : public Overlay {
|
|||
return td::sha256_bits256(td::Slice(buf, 64));
|
||||
}
|
||||
|
||||
bool public_;
|
||||
bool semi_public_ = false;
|
||||
OverlayType overlay_type_;
|
||||
OverlayPrivacyRules rules_;
|
||||
td::string scope_;
|
||||
bool announce_self_ = true;
|
||||
|
|
@ -412,6 +442,25 @@ class OverlayImpl : public Overlay {
|
|||
|
||||
td::ListNode encryptor_lru_;
|
||||
std::map<PublicKeyHash, std::unique_ptr<CachedEncryptor>> encryptor_map_;
|
||||
|
||||
struct PeerList {
|
||||
struct SlaveKey {
|
||||
td::int32 expire_at{0};
|
||||
adnl::AdnlNodeIdShort node{};
|
||||
};
|
||||
using SlaveKeys = std::vector<SlaveKey>;
|
||||
std::map<PublicKeyHash, SlaveKeys> root_public_keys_;
|
||||
OverlayMemberCertificate cert_;
|
||||
std::set<adnl::AdnlNodeIdShort> bad_peers_;
|
||||
adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero();
|
||||
td::DecTree<adnl::AdnlNodeIdShort, OverlayPeer> peers_;
|
||||
std::vector<adnl::AdnlNodeIdShort> neighbours_;
|
||||
|
||||
td::Timestamp local_cert_is_valid_until_;
|
||||
td::uint32 local_member_flags_{0};
|
||||
} peer_list_;
|
||||
|
||||
OverlayOptions opts_;
|
||||
};
|
||||
|
||||
} // namespace overlay
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue