1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00

Accelerator, part 1 (#1119)

This commit contains some parts of https://github.com/ton-blockchain/ton/tree/accelerator
This is auxiliary code that mostly does not change node behavior.

1) Semiprivate overlays and other improvements in overlays code
2) Rename actual_min_split -> monitor_min_split, fix building shard overlays
3) Loading block candidates by block id from DB, fix accept_block after validator restart
4) Cells: ProofStorageStat and changes in CellUsageTree
5) Remove some unused code, other minor changes
This commit is contained in:
SpyCheese 2024-08-23 11:46:40 +03:00 committed by GitHub
parent 9a10f79fba
commit 908415d00b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
66 changed files with 2221 additions and 638 deletions

View file

@ -18,6 +18,7 @@
*/
#include "overlay-manager.h"
#include "auto/tl/ton_api.h"
#include "auto/tl/ton_api.hpp"
#include "overlay.h"
#include "adnl/utils.hpp"
@ -28,9 +29,9 @@
#include "td/db/RocksDb.h"
#include "td/utils/Status.h"
#include "td/utils/buffer.h"
#include "td/utils/overloaded.h"
#include "keys/encryptor.h"
#include "td/utils/port/Poll.h"
#include <vector>
@ -42,13 +43,13 @@ void OverlayManager::update_dht_node(td::actor::ActorId<dht::Dht> dht) {
dht_node_ = dht;
for (auto &X : overlays_) {
for (auto &Y : X.second) {
td::actor::send_closure(Y.second, &Overlay::update_dht_node, dht);
td::actor::send_closure(Y.second.overlay, &Overlay::update_dht_node, dht);
}
}
}
void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id,
td::actor::ActorOwn<Overlay> overlay) {
OverlayMemberCertificate cert, td::actor::ActorOwn<Overlay> overlay) {
auto it = overlays_.find(local_id);
VLOG(OVERLAY_INFO) << this << ": registering overlay " << overlay_id << "@" << local_id;
if (it == overlays_.end()) {
@ -58,19 +59,34 @@ void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdS
td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id,
adnl::Adnl::int_to_bytestring(ton_api::overlay_query::ID),
std::make_unique<AdnlCallback>(actor_id(this)));
td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id,
adnl::Adnl::int_to_bytestring(ton_api::overlay_messageWithExtra::ID),
std::make_unique<AdnlCallback>(actor_id(this)));
td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id,
adnl::Adnl::int_to_bytestring(ton_api::overlay_queryWithExtra::ID),
std::make_unique<AdnlCallback>(actor_id(this)));
}
overlays_[local_id][overlay_id] = std::move(overlay);
overlays_[local_id][overlay_id] = OverlayDescription{std::move(overlay), std::move(cert)};
auto P = td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].get()](td::Result<DbType::GetResult> R) {
R.ensure();
auto value = R.move_as_ok();
if (value.status == td::KeyValue::GetStatus::Ok) {
auto F = fetch_tl_object<ton_api::overlay_db_nodes>(std::move(value.value), true);
F.ensure();
auto nodes = std::move(F.move_as_ok()->nodes_);
td::actor::send_closure(id, &Overlay::receive_nodes_from_db, std::move(nodes));
}
});
auto P =
td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].overlay.get()](td::Result<DbType::GetResult> R) {
R.ensure();
auto value = R.move_as_ok();
if (value.status == td::KeyValue::GetStatus::Ok) {
auto F = fetch_tl_object<ton_api::overlay_db_Nodes>(std::move(value.value), true);
F.ensure();
ton_api::downcast_call(
*F.move_as_ok(), td::overloaded(
[&](ton_api::overlay_db_nodes &V) {
auto nodes = std::move(V.nodes_);
td::actor::send_closure(id, &Overlay::receive_nodes_from_db, std::move(nodes));
},
[&](ton_api::overlay_db_nodesV2 &V) {
auto nodes = std::move(V.nodes_);
td::actor::send_closure(id, &Overlay::receive_nodes_from_db_v2, std::move(nodes));
}));
}
});
auto key = create_hash_tl_object<ton_api::overlay_db_key_nodes>(local_id.bits256_value(), overlay_id.bits256_value());
db_.get(key, std::move(P));
}
@ -84,6 +100,10 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho
adnl::Adnl::int_to_bytestring(ton_api::overlay_message::ID));
td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id,
adnl::Adnl::int_to_bytestring(ton_api::overlay_query::ID));
td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id,
adnl::Adnl::int_to_bytestring(ton_api::overlay_messageWithExtra::ID));
td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id,
adnl::Adnl::int_to_bytestring(ton_api::overlay_queryWithExtra::ID));
overlays_.erase(it);
}
}
@ -101,74 +121,113 @@ void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, Ov
td::string scope, OverlayOptions opts) {
CHECK(!dht_node_.empty());
auto id = overlay_id.compute_short_id();
register_overlay(local_id, id,
Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id),
std::move(callback), std::move(rules), scope, std::move(opts)));
register_overlay(local_id, id, OverlayMemberCertificate{},
Overlay::create_public(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id),
std::move(callback), std::move(rules), scope, std::move(opts)));
}
void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
std::string scope) {
create_private_overlay_ex(local_id, std::move(overlay_id), std::move(nodes), std::move(callback), std::move(rules),
std::move(scope), {});
}
void OverlayManager::create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
std::string scope, OverlayOptions opts) {
auto id = overlay_id.compute_short_id();
register_overlay(local_id, id,
Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id),
std::move(nodes), std::move(callback), std::move(rules), std::move(scope)));
register_overlay(local_id, id, OverlayMemberCertificate{},
Overlay::create_private(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id),
std::move(nodes), std::move(callback), std::move(rules), std::move(scope),
std::move(opts)));
}
void OverlayManager::create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes,
std::vector<PublicKeyHash> root_public_keys,
OverlayMemberCertificate certificate,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope, OverlayOptions opts) {
auto id = overlay_id.compute_short_id();
register_overlay(
local_id, id, certificate,
Overlay::create_semiprivate(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id),
std::move(nodes), std::move(root_public_keys), certificate, std::move(callback),
std::move(rules), std::move(scope), std::move(opts)));
}
void OverlayManager::receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) {
auto R = fetch_tl_prefix<ton_api::overlay_message>(data, true);
if (R.is_error()) {
VLOG(OVERLAY_WARNING) << this << ": can not parse overlay message: " << R.move_as_error();
return;
OverlayIdShort overlay_id;
tl_object_ptr<ton_api::overlay_messageExtra> extra;
auto R = fetch_tl_prefix<ton_api::overlay_messageWithExtra>(data, true);
if (R.is_ok()) {
overlay_id = OverlayIdShort{R.ok()->overlay_};
extra = std::move(R.ok()->extra_);
} else {
auto R2 = fetch_tl_prefix<ton_api::overlay_message>(data, true);
if (R2.is_ok()) {
overlay_id = OverlayIdShort{R2.ok()->overlay_};
} else {
VLOG(OVERLAY_WARNING) << this << ": can not parse overlay message [" << src << "->" << dst
<< "]: " << R2.move_as_error();
return;
}
}
auto M = R.move_as_ok();
auto it = overlays_.find(dst);
if (it == overlays_.end()) {
VLOG(OVERLAY_NOTICE) << this << ": message to unknown overlay " << M->overlay_ << "@" << dst;
VLOG(OVERLAY_NOTICE) << this << ": message to unknown overlay " << overlay_id << "@" << dst;
return;
}
auto it2 = it->second.find(OverlayIdShort{M->overlay_});
auto it2 = it->second.find(overlay_id);
if (it2 == it->second.end()) {
VLOG(OVERLAY_NOTICE) << this << ": message to localid is not in overlay " << M->overlay_ << "@" << dst;
VLOG(OVERLAY_NOTICE) << this << ": message to localid is not in overlay " << overlay_id << "@" << dst;
return;
}
td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false);
td::actor::send_closure(it2->second, &Overlay::receive_message, src, std::move(data));
td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false);
td::actor::send_closure(it2->second.overlay, &Overlay::receive_message, src, std::move(extra), std::move(data));
}
void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data,
td::Promise<td::BufferSlice> promise) {
auto R = fetch_tl_prefix<ton_api::overlay_query>(data, true);
if (R.is_error()) {
VLOG(OVERLAY_WARNING) << this << ": can not parse overlay query [" << src << "->" << dst
<< "]: " << R.move_as_error();
promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad overlay query header"));
return;
OverlayIdShort overlay_id;
tl_object_ptr<ton_api::overlay_messageExtra> extra;
auto R = fetch_tl_prefix<ton_api::overlay_queryWithExtra>(data, true);
if (R.is_ok()) {
overlay_id = OverlayIdShort{R.ok()->overlay_};
extra = std::move(R.ok()->extra_);
} else {
auto R2 = fetch_tl_prefix<ton_api::overlay_query>(data, true);
if (R2.is_ok()) {
overlay_id = OverlayIdShort{R2.ok()->overlay_};
} else {
VLOG(OVERLAY_WARNING) << this << ": can not parse overlay query [" << src << "->" << dst
<< "]: " << R2.move_as_error();
promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad overlay query header"));
return;
}
}
auto M = R.move_as_ok();
auto it = overlays_.find(dst);
if (it == overlays_.end()) {
VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << M->overlay_ << "@" << dst << " from " << src;
VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << overlay_id << "@" << dst << " from " << src;
promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad local_id " << dst));
return;
}
auto it2 = it->second.find(OverlayIdShort{M->overlay_});
auto it2 = it->second.find(overlay_id);
if (it2 == it->second.end()) {
VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " << src;
promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << M->overlay_));
VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << overlay_id << "@" << dst << " from " << src;
promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << overlay_id));
return;
}
td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true);
td::actor::send_closure(it2->second, &Overlay::receive_query, src, std::move(data), std::move(promise));
td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true);
td::actor::send_closure(it2->second.overlay, &Overlay::receive_query, src, std::move(extra), std::move(data),
std::move(promise));
}
void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id,
@ -176,35 +235,64 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS
td::BufferSlice query, td::uint64 max_answer_size,
td::actor::ActorId<adnl::AdnlSenderInterface> via) {
CHECK(query.size() <= adnl::Adnl::huge_packet_max_size());
auto extra = create_tl_object<ton_api::overlay_messageExtra>();
extra->flags_ = 0;
auto it = overlays_.find(src);
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(), true);
td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(),
true);
if (!it2->second.member_certificate.empty()) {
extra->flags_ |= 1;
extra->certificate_ = it2->second.member_certificate.tl();
}
}
}
td::actor::send_closure(
via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise), timeout,
create_serialize_tl_object_suffix<ton_api::overlay_query>(query.as_slice(), overlay_id.tl()), max_answer_size);
auto extra_flags = extra->flags_;
td::BufferSlice serialized_query =
(extra_flags ? create_serialize_tl_object_suffix<ton_api::overlay_queryWithExtra>(
query.as_slice(), overlay_id.tl(), std::move(extra))
: create_serialize_tl_object_suffix<ton_api::overlay_query>(query.as_slice(), overlay_id.tl()));
td::actor::send_closure(via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise),
timeout, std::move(serialized_query), max_answer_size);
}
void OverlayManager::send_message_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id,
td::BufferSlice object, td::actor::ActorId<adnl::AdnlSenderInterface> via) {
CHECK(object.size() <= adnl::Adnl::huge_packet_max_size());
auto extra = create_tl_object<ton_api::overlay_messageExtra>();
extra->flags_ = 0;
auto it = overlays_.find(src);
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(), false);
td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(),
false);
if (!it2->second.member_certificate.empty()) {
// do not send certificate here, we hope that all our neighbours already know of out certificate
// we send it every second to some random nodes. Here we don't want to increase the size of the message
if (false) {
extra->flags_ |= 1;
extra->certificate_ = it2->second.member_certificate.tl();
}
}
}
}
td::actor::send_closure(
via, &adnl::AdnlSenderInterface::send_message, src, dst,
create_serialize_tl_object_suffix<ton_api::overlay_message>(object.as_slice(), overlay_id.tl()));
auto extra_flags = extra->flags_;
td::BufferSlice serialized_message =
(extra_flags ? create_serialize_tl_object_suffix<ton_api::overlay_messageWithExtra>(
object.as_slice(), overlay_id.tl(), std::move(extra))
: create_serialize_tl_object_suffix<ton_api::overlay_message>(object.as_slice(), overlay_id.tl()));
td::actor::send_closure(via, &adnl::AdnlSenderInterface::send_message, src, dst, std::move(serialized_message));
}
void OverlayManager::send_broadcast(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, td::BufferSlice object) {
@ -218,7 +306,7 @@ void OverlayManager::send_broadcast_ex(adnl::AdnlNodeIdShort local_id, OverlayId
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
td::actor::send_closure(it2->second, &Overlay::send_broadcast, send_as, flags, std::move(object));
td::actor::send_closure(it2->second.overlay, &Overlay::send_broadcast, send_as, flags, std::move(object));
}
}
}
@ -235,7 +323,7 @@ void OverlayManager::send_broadcast_fec_ex(adnl::AdnlNodeIdShort local_id, Overl
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
td::actor::send_closure(it2->second, &Overlay::send_broadcast_fec, send_as, flags, std::move(object));
td::actor::send_closure(it2->second.overlay, &Overlay::send_broadcast_fec, send_as, flags, std::move(object));
}
}
}
@ -246,7 +334,7 @@ void OverlayManager::set_privacy_rules(adnl::AdnlNodeIdShort local_id, OverlayId
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
td::actor::send_closure(it2->second, &Overlay::set_privacy_rules, std::move(rules));
td::actor::send_closure(it2->second.overlay, &Overlay::set_privacy_rules, std::move(rules));
}
}
}
@ -257,7 +345,34 @@ void OverlayManager::update_certificate(adnl::AdnlNodeIdShort local_id, OverlayI
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
td::actor::send_closure(it2->second, &Overlay::add_certificate, key, std::move(cert));
td::actor::send_closure(it2->second.overlay, &Overlay::add_certificate, key, std::move(cert));
}
}
}
void OverlayManager::update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id,
OverlayMemberCertificate certificate) {
auto it = overlays_.find(local_id);
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
it2->second.member_certificate = certificate;
td::actor::send_closure(it2->second.overlay, &Overlay::update_member_certificate, certificate);
}
}
}
void OverlayManager::update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes,
std::vector<PublicKeyHash> root_public_keys,
OverlayMemberCertificate certificate) {
auto it = overlays_.find(local_id);
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
it2->second.member_certificate = certificate;
td::actor::send_closure(it2->second.overlay, &Overlay::update_root_member_list, std::move(nodes),
std::move(root_public_keys), std::move(certificate));
}
}
}
@ -266,12 +381,16 @@ void OverlayManager::get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, Ov
td::uint32 max_peers,
td::Promise<std::vector<adnl::AdnlNodeIdShort>> promise) {
auto it = overlays_.find(local_id);
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
td::actor::send_closure(it2->second, &Overlay::get_overlay_random_peers, max_peers, std::move(promise));
}
if (it == overlays_.end()) {
promise.set_error(td::Status::Error(PSTRING() << "no such local id " << local_id));
return;
}
auto it2 = it->second.find(overlay_id);
if (it2 == it->second.end()) {
promise.set_error(td::Status::Error(PSTRING() << "no such overlay " << overlay_id));
return;
}
td::actor::send_closure(it2->second.overlay, &Overlay::get_overlay_random_peers, max_peers, std::move(promise));
}
td::actor::ActorOwn<Overlays> Overlays::create(std::string db_root, td::actor::ActorId<keyring::Keyring> keyring,
@ -334,7 +453,7 @@ void OverlayManager::get_stats(td::Promise<tl_object_ptr<ton_api::engine_validat
for (auto &a : overlays_) {
for (auto &b : a.second) {
td::actor::send_closure(act, &Cb::incr_pending);
td::actor::send_closure(b.second, &Overlay::get_stats,
td::actor::send_closure(b.second.overlay, &Overlay::get_stats,
[act](td::Result<tl_object_ptr<ton_api::engine_validator_overlayStats>> R) {
if (R.is_ok()) {
td::actor::send_closure(act, &Cb::receive_answer, R.move_as_ok());
@ -348,6 +467,19 @@ void OverlayManager::get_stats(td::Promise<tl_object_ptr<ton_api::engine_validat
td::actor::send_closure(act, &Cb::decr_pending);
}
void OverlayManager::forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay,
adnl::AdnlNodeIdShort peer_id) {
auto it = overlays_.find(local_id);
if (it == overlays_.end()) {
return;
}
auto it2 = it->second.find(overlay);
if (it2 == it->second.end()) {
return;
}
td::actor::send_closure(it2->second.overlay, &Overlay::forget_peer, peer_id);
}
Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags,
td::BufferSlice signature)
: issued_by_(issued_by)
@ -454,6 +586,35 @@ tl_object_ptr<ton_api::overlay_Certificate> Certificate::empty_tl() {
return create_tl_object<ton_api::overlay_emptyCertificate>();
}
OverlayMemberCertificate::OverlayMemberCertificate(const ton_api::overlay_MemberCertificate *cert) {
if (!cert) {
expire_at_ = std::numeric_limits<td::int32>::max();
return;
}
if (cert->get_id() == ton_api::overlay_emptyMemberCertificate::ID) {
expire_at_ = std::numeric_limits<td::int32>::max();
return;
}
CHECK(cert->get_id() == ton_api::overlay_memberCertificate::ID);
const auto *real_cert = static_cast<const ton_api::overlay_memberCertificate *>(cert);
signed_by_ = PublicKey(real_cert->issued_by_);
flags_ = real_cert->flags_;
slot_ = real_cert->slot_;
expire_at_ = real_cert->expire_at_;
signature_ = td::SharedSlice(real_cert->signature_.as_slice());
}
td::Status OverlayMemberCertificate::check_signature(const adnl::AdnlNodeIdShort &node) {
if (is_expired()) {
return td::Status::Error(ErrorCode::notready, "certificate is expired");
}
td::BufferSlice data_to_sign = to_sign_data(node);
TRY_RESULT(encryptor, signed_by_.create_encryptor());
TRY_STATUS(encryptor->check_signature(data_to_sign.as_slice(), signature_.as_slice()));
return td::Status::OK();
}
} // namespace overlay
} // namespace ton