1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00

Merge branch 'testnet' into block-generation

# Conflicts:
#	adnl/adnl-query.cpp
#	crypto/block/block.tlb
#	crypto/block/mc-config.h
#	lite-client/lite-client.cpp
#	overlay/overlay-manager.h
#	overlay/overlay-peers.cpp
#	overlay/overlay.cpp
#	overlay/overlay.h
#	overlay/overlay.hpp
#	overlay/overlays.h
#	rldp-http-proxy/DNSResolver.cpp
#	rldp-http-proxy/rldp-http-proxy.cpp
#	tl/generate/scheme/ton_api.tl
#	tl/generate/scheme/ton_api.tlo
#	tl/generate/scheme/tonlib_api.tlo
#	ton/ton-types.h
#	tonlib/tonlib/ExtClient.cpp
#	tonlib/tonlib/ExtClient.h
#	tonlib/tonlib/ExtClientLazy.cpp
#	tonlib/tonlib/ExtClientOutbound.h
#	tonlib/tonlib/ExtClientRaw.h
#	tonlib/tonlib/TonlibClient.cpp
#	tonlib/tonlib/TonlibClient.h
#	tonlib/tonlib/tonlib-cli.cpp
#	validator/impl/collator.cpp
#	validator/impl/validate-query.cpp
#	validator/impl/validate-query.hpp
#	validator/manager.cpp
#	validator/state-serializer.cpp
#	validator/state-serializer.hpp
#	validator/validator-group.cpp
#	validator/validator-group.hpp
#	validator/validator.h
This commit is contained in:
SpyCheese 2022-12-22 23:37:35 +03:00
commit d652f7d706
200 changed files with 13492 additions and 2997 deletions

View file

@ -89,12 +89,20 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho
}
void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) {
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope) {
create_public_overlay_ex(local_id, std::move(overlay_id), std::move(callback), std::move(rules), std::move(scope),
true);
}
void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope, bool announce_self) {
CHECK(!dht_node_.empty());
auto id = overlay_id.compute_short_id();
register_overlay(local_id, id,
Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id),
std::move(callback), std::move(rules), scope));
std::move(callback), std::move(rules), scope, announce_self));
}
void OverlayManager::create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,

View file

@ -52,9 +52,9 @@ class OverlayManager : public Overlays {
void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) override;
void create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope) override;
void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope,
bool announce_self) override;
void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
OverlayPrivacyRules rules) override;

View file

@ -43,6 +43,7 @@ void OverlayImpl::del_peer(adnl::AdnlNodeIdShort id) {
P->set_neighbour(false);
}
peers_.remove(id);
bad_peers_.erase(id);
update_neighbours(0);
}
@ -51,7 +52,16 @@ void OverlayImpl::del_some_peers() {
return;
}
while (peers_.size() > max_peers()) {
auto P = get_random_peer();
OverlayPeer *P;
if (bad_peers_.empty()) {
P = get_random_peer();
} else {
auto it = bad_peers_.upper_bound(next_bad_peer_);
if (it == bad_peers_.end()) {
it = bad_peers_.begin();
}
P = peers_.get(next_bad_peer_ = *it);
}
if (P) {
auto id = P->get_id();
del_peer(id);
@ -118,16 +128,35 @@ void OverlayImpl::add_peer(OverlayNode P) {
add_peer_in(std::move(P));
}
void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::BufferSlice data) {
void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) {
if (!public_) {
return;
}
if (OverlayPeer *p = peers_.get(peer)) {
p->on_ping_result(success);
if (p->is_alive()) {
bad_peers_.erase(peer);
} else {
bad_peers_.insert(peer);
}
}
}
void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result<td::BufferSlice> R) {
CHECK(public_);
auto R = fetch_tl_object<ton_api::overlay_nodes>(std::move(data), true);
on_ping_result(src, R.is_ok());
if (R.is_error()) {
VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeers query: " << R.move_as_error();
return;
}
auto R2 = fetch_tl_object<ton_api::overlay_nodes>(R.move_as_ok(), true);
if (R2.is_error()) {
VLOG(OVERLAY_WARNING) << this << ": dropping incorrect answer to overlay.getRandomPeers query from " << src << ": "
<< R.move_as_error();
<< R2.move_as_error();
return;
}
auto res = R.move_as_ok();
auto res = R2.move_as_ok();
std::vector<OverlayNode> nodes;
for (auto &n : res->nodes_) {
@ -142,12 +171,12 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::BufferSlic
void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode node,
td::Promise<td::BufferSlice> promise) {
std::vector<tl_object_ptr<ton_api::overlay_node>> vec;
if (!is_external_) {
if (announce_self_) {
vec.emplace_back(node.tl());
}
for (td::uint32 i = 0; i < nodes_to_send(); i++) {
auto P = get_random_peer();
auto P = get_random_peer(true);
if (P) {
vec.emplace_back(P->get().tl());
} else {
@ -161,11 +190,7 @@ void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode
} else {
auto P =
td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result<td::BufferSlice> res) {
if (res.is_error()) {
VLOG(OVERLAY_NOTICE) << oid << ": failed getRandomPeers query: " << res.move_as_error();
return;
}
td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers, src, res.move_as_ok());
td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers, src, std::move(res));
});
auto Q =
create_tl_object<ton_api::overlay_getRandomPeers>(create_tl_object<ton_api::overlay_nodes>(std::move(vec)));
@ -218,6 +243,7 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) {
neighbours_.pop_back();
X->set_neighbour(false);
}
bad_peers_.erase(X->get_id());
peers_.remove(X->get_id());
continue;
}
@ -246,15 +272,25 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) {
}
}
OverlayPeer *OverlayImpl::get_random_peer() {
while (peers_.size() > 0) {
OverlayPeer *OverlayImpl::get_random_peer(bool only_alive) {
size_t skip_bad = 3;
while (peers_.size() > (only_alive ? bad_peers_.size() : 0)) {
auto P = peers_.get_random();
if (public_ && P->get_version() + 3600 < td::Clocks::system()) {
VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id();
del_peer(P->get_id());
} else {
return P;
continue;
}
if (!P->is_alive()) {
if (only_alive) {
continue;
}
if (skip_bad > 0) {
--skip_bad;
continue;
}
}
return P;
}
return nullptr;
}
@ -263,17 +299,17 @@ void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers,
td::Promise<std::vector<adnl::AdnlNodeIdShort>> promise) {
std::vector<adnl::AdnlNodeIdShort> v;
auto t = td::Clocks::system();
while (peers_.size() > v.size()) {
while (v.size() < max_peers && v.size() < peers_.size() - bad_peers_.size()) {
auto P = peers_.get_random();
if (P->get_version() + 3600 < t) {
VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id();
del_peer(P->get_id());
} else {
} else if (P->is_alive()) {
bool dup = false;
for (auto &n : v) {
if (n == P->get_id()) {
dup = true;
continue;
break;
}
}
if (!dup) {

View file

@ -38,10 +38,10 @@ td::actor::ActorOwn<Overlay> Overlay::create(td::actor::ActorId<keyring::Keyring
td::actor::ActorId<OverlayManager> manager,
td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
OverlayIdFull overlay_id, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope, bool is_external) {
OverlayPrivacyRules rules, td::string scope, bool announce_self) {
auto R = td::actor::create_actor<OverlayImpl>("overlay", keyring, adnl, manager, dht_node, local_id,
std::move(overlay_id), true, std::vector<adnl::AdnlNodeIdShort>(),
std::move(callback), std::move(rules), scope, is_external);
std::move(callback), std::move(rules), scope, announce_self);
return td::actor::ActorOwn<Overlay>(std::move(R));
}
@ -61,7 +61,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node,
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope, bool is_external)
OverlayPrivacyRules rules, td::string scope, bool announce_self)
: keyring_(keyring)
, adnl_(adnl)
, manager_(manager)
@ -72,15 +72,10 @@ OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor
, public_(pub)
, rules_(std::move(rules))
, scope_(scope)
, is_external_(is_external) {
, announce_self_(announce_self) {
overlay_id_ = id_full_.compute_short_id();
if (is_external_) {
CHECK(public_);
VLOG(OVERLAY_INFO) << this << ": creating public external";
} else {
VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private");
}
VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private");
for (auto &node : nodes) {
CHECK(!public_);
@ -139,13 +134,6 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getB
}
void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise<td::BufferSlice> promise) {
if (is_external_) {
LOG(OVERLAY_WARNING) << "dropping query in external overlay " << overlay_id_;
promise.set_error(td::Status::Error("overlay is external"));
td::actor::send_closure(manager_, &Overlays::send_message, src, local_id_, overlay_id_,
create_serialize_tl_object<ton_api::overlay_message_removePeer>());
return;
}
if (!public_) {
auto P = peers_.get(src);
if (P == nullptr) {
@ -153,6 +141,8 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data,
promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is private"));
return;
}
} else {
on_ping_result(src, true);
}
auto R = fetch_tl_object<ton_api::Function>(data.clone(), true);
@ -231,24 +221,10 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat
VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src;
return;
}
} else {
on_ping_result(src, true);
}
auto X = fetch_tl_object<ton_api::overlay_Broadcast>(data.clone(), true);
if (X.is_error()) {
auto Y = fetch_tl_object<ton_api::overlay_message_removePeer>(data.clone(), true);
if (Y.is_ok() && public_) {
VLOG(OVERLAY_DEBUG) << this << ": received removePeer message from " << src;
if (peers_.exists(src)) {
del_peer(src);
}
callback_->on_remove_peer(src);
return;
}
}
if (is_external_) {
LOG(OVERLAY_WARNING) << "dropping message in external overlay " << overlay_id_;
return;
}
if (X.is_error()) {
VLOG(OVERLAY_DEBUG) << this << ": received custom message";
callback_->receive_message(src, overlay_id_, std::move(data));
@ -349,7 +325,7 @@ void OverlayImpl::receive_dht_nodes(td::Result<dht::DhtValue> res, bool dummy) {
VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << res.move_as_error();
}
if (is_external_) {
if (!announce_self_) {
return;
}
@ -555,9 +531,6 @@ void OverlayImpl::send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::B
}
void OverlayImpl::deliver_broadcast(PublicKeyHash source, td::BufferSlice data) {
if (is_external_) {
return;
}
callback_->receive_broadcast(source, overlay_id_, std::move(data));
}
@ -630,10 +603,6 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) {
}
void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise<td::Unit> promise) {
if (is_external_) {
promise.set_result(td::Unit());
return;
}
callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise));
}

View file

@ -42,7 +42,7 @@ class Overlay : public td::actor::Actor {
td::actor::ActorId<OverlayManager> manager,
td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
OverlayIdFull overlay_id, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope, bool is_external = false);
OverlayPrivacyRules rules, td::string scope, bool announce_self = true);
static td::actor::ActorOwn<Overlay> create(td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<OverlayManager> manager,

View file

@ -79,7 +79,17 @@ class OverlayPeer {
td::int32 get_version() const {
return node_.version();
}
void on_ping_result(bool success) {
if (success) {
missed_pings_ = 0;
} else {
++missed_pings_;
}
}
bool is_alive() const {
return missed_pings_ < 3;
}
td::uint32 throughput_out_bytes = 0;
td::uint32 throughput_in_bytes = 0;
@ -105,6 +115,7 @@ class OverlayPeer {
adnl::AdnlNodeIdShort id_;
bool is_neighbour_ = false;
size_t missed_pings_ = 0;
};
class OverlayImpl : public Overlay {
@ -113,7 +124,7 @@ class OverlayImpl : public Overlay {
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node,
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", bool is_external = false);
OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", bool announce_self = true);
void update_dht_node(td::actor::ActorId<dht::Dht> dht) override {
dht_node_ = dht;
}
@ -138,7 +149,8 @@ class OverlayImpl : public Overlay {
alarm_timestamp() = td::Timestamp::in(1);
}
void receive_random_peers(adnl::AdnlNodeIdShort src, td::BufferSlice data);
void on_ping_result(adnl::AdnlNodeIdShort peer, bool success);
void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result<td::BufferSlice> R);
void send_random_peers(adnl::AdnlNodeIdShort dst, td::Promise<td::BufferSlice> promise);
void send_random_peers_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise<td::BufferSlice> promise);
void get_overlay_random_peers(td::uint32 max_peers, td::Promise<std::vector<adnl::AdnlNodeIdShort>> promise) override;
@ -299,7 +311,7 @@ class OverlayImpl : public Overlay {
void add_peers(std::vector<OverlayNode> nodes);
void del_some_peers();
void del_peer(adnl::AdnlNodeIdShort id);
OverlayPeer *get_random_peer();
OverlayPeer *get_random_peer(bool only_alive = false);
td::actor::ActorId<keyring::Keyring> keyring_;
td::actor::ActorId<adnl::Adnl> adnl_;
@ -314,6 +326,8 @@ class OverlayImpl : public Overlay {
td::Timestamp update_db_at_;
td::Timestamp update_throughput_at_;
td::Timestamp last_throughput_update_;
std::set<adnl::AdnlNodeIdShort> bad_peers_;
adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero();
std::unique_ptr<Overlays::Callback> callback_;
@ -371,7 +385,7 @@ class OverlayImpl : public Overlay {
bool semi_public_ = false;
OverlayPrivacyRules rules_;
td::string scope_;
bool is_external_ = false;
bool announce_self_ = true;
std::map<PublicKeyHash, std::shared_ptr<Certificate>> certs_;
class CachedEncryptor : public td::ListNode {

View file

@ -195,10 +195,11 @@ class Overlays : public td::actor::Actor {
virtual void update_dht_node(td::actor::ActorId<dht::Dht> dht) = 0;
virtual void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) = 0;
virtual void create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope) = 0;
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope) = 0;
virtual void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope, bool announce_self) = 0;
virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
OverlayPrivacyRules rules) = 0;