From 0e428af80795db9906d2665ecce3ab31474c2a87 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 6 Oct 2022 15:13:10 +0300 Subject: [PATCH] Tonlib uses partial liteservers --- tl/generate/scheme/tonlib_api.tl | 2 +- tl/generate/scheme/tonlib_api.tlo | Bin 30836 -> 30904 bytes tonlib/CMakeLists.txt | 4 +- tonlib/tonlib/Config.cpp | 51 +++-- tonlib/tonlib/Config.h | 2 + tonlib/tonlib/ExtClient.cpp | 6 +- tonlib/tonlib/ExtClient.h | 15 +- tonlib/tonlib/ExtClientLazy.cpp | 24 +-- tonlib/tonlib/ExtClientMulti.cpp | 169 +++++++++++++++ tonlib/tonlib/ExtClientMulti.h | 61 ++++++ tonlib/tonlib/ExtClientOutbound.cpp | 8 +- tonlib/tonlib/ExtClientOutbound.h | 6 +- .../{ExtClientLazy.h => ExtClientRaw.h} | 15 +- tonlib/tonlib/QueryTraits.h | 202 ++++++++++++++++++ tonlib/tonlib/TonlibClient.cpp | 38 ++-- tonlib/tonlib/TonlibClient.h | 4 +- tonlib/tonlib/tonlib-cli.cpp | 31 ++- 17 files changed, 555 insertions(+), 83 deletions(-) create mode 100644 tonlib/tonlib/ExtClientMulti.cpp create mode 100644 tonlib/tonlib/ExtClientMulti.h rename tonlib/tonlib/{ExtClientLazy.h => ExtClientRaw.h} (68%) create mode 100644 tonlib/tonlib/QueryTraits.h diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 79185d5b..e83ee784 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -183,7 +183,7 @@ smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc. smc.libraryEntry hash:int256 data:bytes = smc.LibraryEntry; smc.libraryResult result:(vector smc.libraryEntry) = smc.LibraryResult; -updateSendLiteServerQuery id:int64 data:bytes = Update; +updateSendLiteServerQuery id:int64 data:bytes workchain:int32 shard:int64 = Update; updateSyncState sync_state:SyncState = Update; //@class LogStream @description Describes a stream to which tonlib internal log is written diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index ee457609da33cca7655480b0531417fd19ccf958..5f3970a1197f2a1c2cdabfbcb6e8638fcc202586 100644 GIT binary patch delta 94 zcmezJfpN!2#tjSDc<1 Config::parse(std::string str) { if (json.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (1)"); } - //TRY_RESULT(main_type, td::get_json_object_string_field(json.get_object(), "@type", false)); - //if (main_type != "config.global") { - //return td::Status::Error("Invalid config (3)"); - //} + td::JsonArray empty_array; TRY_RESULT(lite_clients_obj, - td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, false)); - auto &lite_clients = lite_clients_obj.get_array(); + td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, true)); + auto &lite_clients = + lite_clients_obj.type() == td::JsonValue::Type::Array ? lite_clients_obj.get_array() : empty_array; + TRY_RESULT(lite_clients_v2_obj, + td::get_json_object_field(json.get_object(), "liteservers_v2", td::JsonValue::Type::Array, true)); + auto &lite_clients_v2 = + lite_clients_v2_obj.type() == td::JsonValue::Type::Array ? lite_clients_v2_obj.get_array() : empty_array; - Config res; - for (auto &value : lite_clients) { + auto parse_desc = [&](td::JsonValue& value) -> td::Result { if (value.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (2)"); } auto &object = value.get_object(); - //TRY_RESULT(value_type, td::get_json_object_string_field(object, "@type", false)); - //if (value_type != "liteclient.config.global") { - //return td::Status::Error("Invalid config (4)"); - //} TRY_RESULT(ip, td::get_json_object_long_field(object, "ip", false)); TRY_RESULT(port, td::get_json_object_int_field(object, "port", false)); @@ -93,15 +90,41 @@ td::Result Config::parse(std::string str) { auto &id = id_obj.get_object(); TRY_RESULT(id_type, td::get_json_object_string_field(id, "@type", false)); if (id_type != "pub.ed25519") { - return td::Status::Error("Invalid config (5)"); + return td::Status::Error("Invalid config (3)"); } TRY_RESULT(key_base64, td::get_json_object_string_field(id, "key", false)); TRY_RESULT(key, td::base64_decode(key_base64)); if (key.size() != 32) { - return td::Status::Error("Invalid config (6)"); + return td::Status::Error("Invalid config (4)"); } client.adnl_id = ton::adnl::AdnlNodeIdFull(ton::pubkeys::Ed25519(td::Bits256(td::Slice(key).ubegin()))); + return client; + }; + + Config res; + for (auto &value : lite_clients) { + TRY_RESULT(client, parse_desc(value)); + res.lite_clients.push_back(std::move(client)); + } + for (auto &value : lite_clients_v2) { + TRY_RESULT(client, parse_desc(value)); + client.is_full = false; + TRY_RESULT(shards_obj, + td::get_json_object_field(value.get_object(), "shards", td::JsonValue::Type::Array, false)); + for (auto &shard : shards_obj.get_array()) { + if (shard.type() != td::JsonValue::Type::Object) { + return td::Status::Error("Invalid config (5)"); + } + auto &shard_obj = shard.get_object(); + TRY_RESULT(workchain, td::get_json_object_int_field(shard_obj, "workchain", false)); + TRY_RESULT(shard_id, td::get_json_object_long_field(shard_obj, "shard", false)); + if (shard_id == 0) { + return td::Status::Error("Invalid config (6)"); + } + client.shards.emplace_back(workchain, shard_id); + } + res.lite_clients.push_back(std::move(client)); } diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 3902c341..420889df 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -26,6 +26,8 @@ struct Config { struct LiteClient { ton::adnl::AdnlNodeIdFull adnl_id; td::IPAddress address; + bool is_full = true; + std::vector shards; }; ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 30a29b59..fdd1606d 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -54,7 +54,7 @@ void ExtClient::with_last_block(td::Promise promise) { td::actor::send_closure(client_.last_block_actor_, &LastBlock::get_last_block, std::move(P)); } -void ExtClient::send_raw_query(td::BufferSlice query, td::Promise promise) { +void ExtClient::send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) { auto query_id = queries_.create(std::move(promise)); td::Promise P = [query_id, self = this, actor_id = td::actor::actor_id()](td::Result result) { @@ -62,10 +62,10 @@ void ExtClient::send_raw_query(td::BufferSlice query, td::Promisequeries_.extract(query_id).set_result(std::move(result)); }); }; - if (client_.adnl_ext_client_.empty()) { + if (client_.raw_client_.empty()) { return P.set_error(TonlibError::NoLiteServers()); } - td::actor::send_closure(client_.adnl_ext_client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(query), + td::actor::send_closure(client_.raw_client_, &ExtClientRaw::send_query, "query", std::move(query), shard, td::Timestamp::in(10.0), std::move(P)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index 707ca74b..48818622 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -28,9 +28,11 @@ #include "td/utils/Container.h" #include "td/utils/Random.h" -#include "ExtClientLazy.h" +#include "ExtClientRaw.h" #include "TonlibError.h" #include "utils.h" +#include "QueryTraits.h" +#include "ExtClientRaw.h" namespace tonlib { class LastBlock; @@ -38,7 +40,7 @@ class LastConfig; struct LastBlockState; struct LastConfigState; struct ExtClientRef { - td::actor::ActorId adnl_ext_client_; + td::actor::ActorId raw_client_; td::actor::ActorId last_block_actor_; td::actor::ActorId last_config_actor_; }; @@ -64,6 +66,7 @@ class ExtClient { template void send_query(QueryT query, td::Promise promise, td::int32 seq_no = -1) { + ton::ShardIdFull shard = QueryTraits::get_shard(query); auto raw_query = ton::serialize_tl_object(&query, true); td::uint32 tag = td::Random::fast_uint32(); VLOG(lite_server) << "send query to liteserver: " << tag << " " << to_string(query); @@ -77,7 +80,7 @@ class ExtClient { ton::serialize_tl_object(ton::create_tl_object(std::move(raw_query)), true); send_raw_query( - std::move(liteserver_query), [promise = std::move(promise), tag](td::Result R) mutable { + std::move(liteserver_query), shard, [promise = std::move(promise), tag](td::Result R) mutable { auto res = [&]() -> td::Result { TRY_RESULT_PREFIX(data, std::move(R), TonlibError::LiteServerNetwork()); auto r_error = ton::fetch_tl_object(data.clone(), true); @@ -96,8 +99,8 @@ class ExtClient { } void force_change_liteserver() { - if (!client_.adnl_ext_client_.empty()) { - td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::force_change_liteserver); + if (!client_.raw_client_.empty()) { + td::actor::send_closure(client_.raw_client_, &ExtClientRaw::force_change_liteserver); } } @@ -107,6 +110,6 @@ class ExtClient { td::Container> last_block_queries_; td::Container> last_config_queries_; - void send_raw_query(td::BufferSlice query, td::Promise promise); + void send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise); }; } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientLazy.cpp b/tonlib/tonlib/ExtClientLazy.cpp index 335a0ff9..625d7eaa 100644 --- a/tonlib/tonlib/ExtClientLazy.cpp +++ b/tonlib/tonlib/ExtClientLazy.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "ExtClientLazy.h" +#include "ExtClientRaw.h" #include "TonlibError.h" #include "td/utils/Random.h" namespace tonlib { -class ExtClientLazyImp : public ExtClientLazy { +class ExtClientLazyImp : public ExtClientRaw { public: ExtClientLazyImp(std::vector> servers, - td::unique_ptr callback) + td::unique_ptr callback) : servers_(std::move(servers)), callback_(std::move(callback)) { CHECK(!servers_.empty()); } @@ -34,15 +34,7 @@ class ExtClientLazyImp : public ExtClientLazy { td::random_shuffle(td::as_mutable_span(servers_), rnd); } - void check_ready(td::Promise promise) override { - before_query(); - if (client_.empty()) { - return promise.set_error(TonlibError::Cancelled()); - } - send_closure(client_, &ton::adnl::AdnlExtClient::check_ready, std::move(promise)); - } - - void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, td::Promise promise) override { before_query(); if (client_.empty()) { @@ -109,7 +101,7 @@ class ExtClientLazyImp : public ExtClientLazy { bool cur_server_bad_force_ = false; td::actor::ActorOwn client_; - td::unique_ptr callback_; + td::unique_ptr callback_; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; bool is_closing_{false}; @@ -140,12 +132,12 @@ class ExtClientLazyImp : public ExtClientLazy { } }; -td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback) { +td::actor::ActorOwn ExtClientRaw::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback) { return create({std::make_pair(dst, dst_addr)}, std::move(callback)); } -td::actor::ActorOwn ExtClientLazy::create( +td::actor::ActorOwn ExtClientRaw::create( std::vector> servers, td::unique_ptr callback) { return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); } diff --git a/tonlib/tonlib/ExtClientMulti.cpp b/tonlib/tonlib/ExtClientMulti.cpp new file mode 100644 index 00000000..3a6cdf59 --- /dev/null +++ b/tonlib/tonlib/ExtClientMulti.cpp @@ -0,0 +1,169 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "ExtClientMulti.h" +#include "ton/ton-shard.h" +#include "td/utils/Random.h" + +namespace tonlib { + +static const double MAX_NO_QUERIES_TIMEOUT = 120; + +ExtClientMulti::ExtClientMulti(std::vector clients, td::unique_ptr callback) + : callback_(std::move(callback)) { + for (auto &desc : clients) { + servers_.emplace_back(); + servers_.back().desc = std::move(desc); + } +} + +void ExtClientMulti::start_up() { + alarm_timestamp() = td::Timestamp::in(60.0); +} + +void ExtClientMulti::send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + td::Promise promise) { + if (shard.is_masterchain() && mc_server_idx_ != -1 && !servers_[mc_server_idx_].is_bad()) { + send_query_to_server(std::move(name), std::move(data), mc_server_idx_, timeout, std::move(promise)); + return; + } + auto it = shard_server_idx_cached_.find(shard); + if (it != shard_server_idx_cached_.end() && !servers_[it->second].is_bad()) { + send_query_to_server(std::move(name), std::move(data), it->second, timeout, std::move(promise)); + return; + } + int server_idx = -1; + int random_idx = -1; + int cnt = 0; + int best_prefix = -1; + for (int i = 0; i < (int)servers_.size(); ++i) { + const Server &server = servers_[i]; + if (server.is_bad()) { + continue; + } + int len = server.desc.is_full ? 65 : server.max_supported_prefix(shard); + if (len > best_prefix) { + best_prefix = len; + server_idx = -1; + random_idx = -1; + cnt = 0; + } else if (len < best_prefix) { + continue; + } + if (!server.client.empty()) { + server_idx = i; + } + if (td::Random::fast(0, cnt) == 0) { + random_idx = i; + } + ++cnt; + } + if (server_idx == -1) { + server_idx = random_idx; + } + if (server_idx == -1) { + promise.set_error(td::Status::Error("failed to select a suitable server")); + return; + } + if (shard.pfx_len() <= ton::max_shard_pfx_len) { + shard_server_idx_cached_[shard] = server_idx; + } + if (shard.is_masterchain() || servers_[server_idx].desc.is_full) { + mc_server_idx_ = server_idx; + } + send_query_to_server(std::move(name), std::move(data), server_idx, timeout, std::move(promise)); +} + +void ExtClientMulti::send_query_to_server(std::string name, td::BufferSlice data, int server_idx, td::Timestamp timeout, + td::Promise promise) { + Server &server = servers_.at(server_idx); + if (server.client.empty()) { + start_client(server_idx); + } + server.ttl = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); + td::Promise P = [SelfId = actor_id(this), idx = server_idx, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error() && + (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { + td::actor::send_closure(SelfId, &ExtClientMulti::set_server_bad, idx); + } + promise.set_result(std::move(R)); + }; + send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, + std::move(P)); +} + +void ExtClientMulti::force_change_liteserver() { + if (mc_server_idx_ != -1) { + set_server_bad(mc_server_idx_); + mc_server_idx_ = -1; + } +} + +void ExtClientMulti::start_client(int server_idx) { + class Callback : public ton::adnl::AdnlExtClient::Callback { + public: + Callback(td::actor::ActorId parent, int idx) : parent_(std::move(parent)), idx_(idx) { + } + void on_ready() override { + } + void on_stop_ready() override { + td::actor::send_closure(parent_, &ExtClientMulti::set_server_bad, idx_); + } + + private: + td::actor::ActorId parent_; + int idx_; + }; + Server &server = servers_.at(server_idx); + server.client = ton::adnl::AdnlExtClient::create(server.desc.adnl_id, server.desc.address, + std::make_unique(actor_id(this), server_idx)); +} + +void ExtClientMulti::alarm() { + for (Server& server : servers_) { + if (server.ttl && server.ttl.is_in_past()) { + server.client.reset(); + } + } + alarm_timestamp() = td::Timestamp::in(60.0); +} + +void ExtClientMulti::set_server_bad(int idx) { + Server& server = servers_.at(idx); + server.client.reset(); + server.ttl = td::Timestamp::never(); + server.ignore_until = td::Timestamp::in(10.0); +} + +int ExtClientMulti::Server::max_supported_prefix(ton::ShardIdFull shard) const { + if (desc.is_full || shard.is_masterchain()) { + return shard.pfx_len(); + } + int res = -1; + for (const ton::ShardIdFull &our_shard : desc.shards) { + if (ton::shard_is_ancestor(our_shard, shard)) { + return shard.pfx_len(); + } + if (shard.workchain == our_shard.workchain) { + int x = std::min({shard.pfx_len(), our_shard.pfx_len(), ton::count_matching_bits(shard.shard, our_shard.shard)}); + res = std::max(res, x); + } + } + return res; +} + +} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientMulti.h b/tonlib/tonlib/ExtClientMulti.h new file mode 100644 index 00000000..4c93b00e --- /dev/null +++ b/tonlib/tonlib/ExtClientMulti.h @@ -0,0 +1,61 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "td/actor/actor.h" +#include "adnl/adnl-ext-client.h" +#include "Config.h" +#include "ExtClientRaw.h" +#include + +namespace tonlib { + +class ExtClientMulti : public ExtClientRaw { + public: + ExtClientMulti(std::vector clients, td::unique_ptr callback); + + void start_up() override; + void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + td::Promise promise) override; + void alarm() override; + void force_change_liteserver() override; + + private: + void send_query_to_server(std::string name, td::BufferSlice data, int server_idx, td::Timestamp timeout, + td::Promise promise); + void start_client(int server_idx); + + struct Server { + Config::LiteClient desc; + td::actor::ActorOwn client; + td::Timestamp ttl; + td::Timestamp ignore_until = td::Timestamp::never(); + + int max_supported_prefix(ton::ShardIdFull shard) const; + bool is_bad() const { + return ignore_until && !ignore_until.is_in_past(); + } + }; + + void set_server_bad(int idx); + + td::unique_ptr callback_; + std::vector servers_; + int mc_server_idx_ = -1; + std::map shard_server_idx_cached_; +}; + +} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index 025ba848..e057368d 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -27,15 +27,11 @@ class ExtClientOutboundImp : public ExtClientOutbound { ExtClientOutboundImp(td::unique_ptr callback) : callback_(std::move(callback)) { } - void check_ready(td::Promise promise) override { - promise.set_error(td::Status::Error("Not supported")); - } - - void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, td::Promise promise) override { auto query_id = next_query_id_++; queries_[query_id] = std::move(promise); - callback_->request(query_id, data.as_slice().str()); + callback_->request(query_id, data.as_slice().str(), shard); } void force_change_liteserver() override { diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 87b73b9e..91217868 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -19,16 +19,16 @@ #pragma once #include "td/actor/actor.h" -#include "ExtClientLazy.h" +#include "ExtClientRaw.h" namespace tonlib { -class ExtClientOutbound : public ExtClientLazy { +class ExtClientOutbound : public ExtClientRaw { public: class Callback { public: virtual ~Callback() { } - virtual void request(td::int64 id, std::string data) = 0; + virtual void request(td::int64 id, std::string data, ton::ShardIdFull shard) = 0; }; virtual void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) = 0; static td::actor::ActorOwn create(td::unique_ptr callback); diff --git a/tonlib/tonlib/ExtClientLazy.h b/tonlib/tonlib/ExtClientRaw.h similarity index 68% rename from tonlib/tonlib/ExtClientLazy.h rename to tonlib/tonlib/ExtClientRaw.h index dc4490b3..0a55bdc1 100644 --- a/tonlib/tonlib/ExtClientLazy.h +++ b/tonlib/tonlib/ExtClientRaw.h @@ -18,23 +18,24 @@ */ #pragma once #include "td/actor/actor.h" - #include "adnl/adnl-ext-client.h" +#include "ton/ton-types.h" namespace tonlib { -class ExtClientLazy : public ton::adnl::AdnlExtClient { +class ExtClientRaw : public td::actor::Actor { public: class Callback { public: - virtual ~Callback() { - } + virtual ~Callback() = default; }; + virtual void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + td::Promise promise) = 0; virtual void force_change_liteserver() = 0; - static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback); - static td::actor::ActorOwn create( + static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback); + static td::actor::ActorOwn create( std::vector> servers, td::unique_ptr callback); }; diff --git a/tonlib/tonlib/QueryTraits.h b/tonlib/tonlib/QueryTraits.h new file mode 100644 index 00000000..bf480f64 --- /dev/null +++ b/tonlib/tonlib/QueryTraits.h @@ -0,0 +1,202 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "auto/tl/lite_api.h" +#include "vm/boc.h" +#include "vm/cellslice.h" +#include "block/block-auto.h" +#include "block/block-parse.h" + +namespace tonlib { + +template +struct QueryTraits { + static ton::ShardIdFull get_shard(const Query& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getMasterchainInfo& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getMasterchainInfoExt& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getTime& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getVersion& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlock& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getState& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockHeader& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_sendMessage& q) { + auto shard = [&]() -> td::Result { + vm::BagOfCells boc; + TRY_STATUS(boc.deserialize(q.body_.as_slice())); + if (boc.get_root_count() != 1) { + return td::Status::Error("external message is not a valid bag of cells"); + } + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + if (!tlb::unpack_cell_inexact(boc.get_root_cell(), info)) { + return td::Status::Error("cannot unpack external message header"); + } + auto dest_prefix = block::tlb::t_MsgAddressInt.get_prefix(info.dest); + if (!dest_prefix.is_valid()) { + return td::Status::Error("destination of an inbound external message is an invalid blockchain address"); + } + return dest_prefix.as_leaf_shard(); + }(); + if (shard.is_error()) { + LOG(DEBUG) << "Failed to get shard from query liteServer.sendMessage: " << shard.move_as_error(); + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } + return shard.move_as_ok(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAccountState& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_runSmcMethod& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getShardInfo& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAllShardsInfo& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getOneTransaction& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getTransactions& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_lookupBlock& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_listBlockTransactions& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockProof& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getConfigAll& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getConfigParams& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getValidatorStats& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getLibraries& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + + +} // namespace tonlib diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index f9e984bb..b4b38697 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -18,7 +18,7 @@ */ #include "TonlibClient.h" -#include "tonlib/ExtClientLazy.h" +#include "tonlib/ExtClientRaw.h" #include "tonlib/ExtClientOutbound.h" #include "tonlib/LastBlock.h" #include "tonlib/LastConfig.h" @@ -27,6 +27,7 @@ #include "tonlib/keys/Mnemonic.h" #include "tonlib/keys/SimpleEncryption.h" #include "tonlib/TonlibError.h" +#include "tonlib/ExtClientMulti.h" #include "smc-envelope/GenericAccount.h" #include "smc-envelope/ManualDns.h" @@ -1649,15 +1650,16 @@ void TonlibClient::hangup() { ExtClientRef TonlibClient::get_client_ref() { ExtClientRef ref; - ref.adnl_ext_client_ = raw_client_.get(); + ref.raw_client_ = raw_client_.get(); ref.last_block_actor_ = raw_last_block_.get(); ref.last_config_actor_ = raw_last_config_.get(); return ref; } -void TonlibClient::proxy_request(td::int64 query_id, std::string data) { - on_update(tonlib_api::make_object(query_id, data)); +void TonlibClient::proxy_request(td::int64 query_id, std::string data, ton::ShardIdFull shard) { + on_update( + tonlib_api::make_object(query_id, data, shard.workchain, shard.shard)); } void TonlibClient::init_ext_client() { @@ -1668,9 +1670,9 @@ void TonlibClient::init_ext_client() { : parent_(std::move(parent)), config_generation_(config_generation) { } - void request(td::int64 id, std::string data) override { + void request(td::int64 id, std::string data, ton::ShardIdFull shard) override { send_closure(parent_, &TonlibClient::proxy_request, (id << 16) | (config_generation_ & 0xffff), - std::move(data)); + std::move(data), shard); } private: @@ -1683,21 +1685,29 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - std::vector> servers; - for (const auto& s : config_.lite_clients) { - servers.emplace_back(s.adnl_id, s.address); - } - class Callback : public ExtClientLazy::Callback { + class Callback : public ExtClientRaw::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } - private: td::actor::ActorShared<> parent_; }; + std::vector> full_servers; + for (const auto& s : config_.lite_clients) { + if (s.is_full) { + full_servers.emplace_back(s.adnl_id, s.address); + } + } + if (!full_servers.empty()) { + raw_client_ = + ExtClientRaw::create(std::move(full_servers), td::make_unique(td::actor::actor_shared())); + } else { + CHECK(!config_.lite_clients.empty()); + raw_client_ = td::actor::create_actor("ExtClientMulti", config_.lite_clients, + td::make_unique(td::actor::actor_shared())); + } ext_client_outbound_ = {}; ref_cnt_++; - raw_client_ = ExtClientLazy::create(std::move(servers), td::make_unique(td::actor::actor_shared())); } } @@ -2826,7 +2836,7 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request, auto actor_id = actor_id_++; actors_[actor_id] = td::actor::create_actor( - "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), + "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), promise.wrap([private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable { return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x)); })); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index e7819290..8c076876 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -110,7 +110,7 @@ class TonlibClient : public td::actor::Actor { vm::Dictionary libraries{256}; // network - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; td::actor::ActorId ext_client_outbound_; td::actor::ActorOwn raw_last_block_; td::actor::ActorOwn raw_last_config_; @@ -372,7 +372,7 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::getConfigParam& request, td::Promise>&& promise); - void proxy_request(td::int64 query_id, std::string data); + void proxy_request(td::int64 query_id, std::string data, ton::ShardIdFull shard); void load_libs_from_disk(); void store_libs_to_disk(); diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 0a042eb1..c256bf1e 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -47,7 +47,7 @@ #include "tonlib/TonlibClient.h" #include "tonlib/TonlibCallback.h" -#include "tonlib/ExtClientLazy.h" +#include "tonlib/ExtClientRaw.h" #include "smc-envelope/ManualDns.h" #include "smc-envelope/PaymentChannel.h" @@ -62,6 +62,7 @@ #include #include #include "git.h" +#include "ExtClientMulti.h" using tonlib_api::make_object; @@ -174,7 +175,7 @@ class TonlibCli : public td::actor::Actor { std::map>> query_handlers_; - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; bool is_closing_{false}; td::uint32 ref_cnt_{1}; @@ -223,11 +224,7 @@ class TonlibCli : public td::actor::Actor { if (options_.use_callbacks_for_network) { auto config = tonlib::Config::parse(options_.config).move_as_ok(); - auto lite_clients_size = config.lite_clients.size(); - CHECK(lite_clients_size != 0); - auto lite_client_id = td::Random::fast(0, td::narrow_cast(lite_clients_size) - 1); - auto& lite_client = config.lite_clients[lite_client_id]; - class Callback : public tonlib::ExtClientLazy::Callback { + class Callback : public tonlib::ExtClientRaw::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } @@ -235,9 +232,22 @@ class TonlibCli : public td::actor::Actor { private: td::actor::ActorShared<> parent_; }; + std::vector> full_servers; + int lite_client_id = -1, cnt = 0; + for (const auto& s : config.lite_clients) { + if (s.is_full) { + full_servers.emplace_back(s.adnl_id, s.address); + } + } + if (!full_servers.empty()) { + raw_client_ = tonlib::ExtClientRaw::create(std::move(full_servers), + td::make_unique(td::actor::actor_shared())); + } else { + CHECK(!config.lite_clients.empty()); + raw_client_ = td::actor::create_actor( + "ExtClientMulti", config.lite_clients, td::make_unique(td::actor::actor_shared())); + } ref_cnt_++; - raw_client_ = tonlib::ExtClientLazy::create(lite_client.adnl_id, lite_client.address, - td::make_unique(td::actor::actor_shared())); } auto config = !options_.config.empty() @@ -1534,7 +1544,8 @@ class TonlibCli : public td::actor::Actor { auto update = tonlib_api::move_object_as(std::move(result)); CHECK(!raw_client_.empty()); snd_bytes_ += update->data_.size(); - send_closure(raw_client_, &ton::adnl::AdnlExtClient::send_query, "query", td::BufferSlice(update->data_), + ton::ShardIdFull shard(update->workchain_, update->shard_); + send_closure(raw_client_, &tonlib::ExtClientRaw::send_query, "query", td::BufferSlice(update->data_), shard, td::Timestamp::in(5), [actor_id = actor_id(this), id = update->id_](td::Result res) { send_closure(actor_id, &TonlibCli::on_adnl_result, id, std::move(res));