1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00

Accelerator: partial fullnodes (#1393)

* Accelerator: partial fullnodes

1) Node can monitor a subset of shards
2) New archive slice format (sharded)
3) Validators are still required to have all shards
4) Support partial liteservers in lite-client, blockchain explorer, tonlib
5) Proxy liteserver

* Fix compilation error
This commit is contained in:
SpyCheese 2024-11-26 15:46:58 +04:00 committed by GitHub
parent 62444100f5
commit 954a96a077
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
83 changed files with 3213 additions and 1113 deletions

View file

@ -1,9 +1,10 @@
cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h)
add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h
query-utils.hpp query-utils.cpp)
target_link_libraries(lite-client-common PUBLIC tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto)
add_executable(lite-client lite-client.cpp lite-client.h)
add_executable(lite-client lite-client.cpp lite-client.h ext-client.h ext-client.cpp)
target_link_libraries(lite-client tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils terminal lite-client-common git)
install(TARGETS lite-client RUNTIME DESTINATION bin)

228
lite-client/ext-client.cpp Normal file
View file

@ -0,0 +1,228 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ext-client.h"
#include "td/utils/Random.h"
#include "ton/ton-shard.h"
namespace liteclient {
class ExtClientImpl : public ExtClient {
public:
ExtClientImpl(std::vector<LiteServerConfig> liteservers, td::unique_ptr<Callback> callback, bool connect_to_all)
: callback_(std::move(callback)), connect_to_all_(connect_to_all) {
CHECK(!liteservers.empty());
servers_.resize(liteservers.size());
for (size_t i = 0; i < servers_.size(); ++i) {
servers_[i].config = std::move(liteservers[i]);
servers_[i].idx = i;
}
}
void start_up() override {
LOG(INFO) << "Started ext client, " << servers_.size() << " liteservers";
td::Random::Fast rnd;
td::random_shuffle(td::as_mutable_span(servers_), rnd);
server_indices_.resize(servers_.size());
for (size_t i = 0; i < servers_.size(); ++i) {
server_indices_[servers_[i].idx] = i;
}
if (connect_to_all_) {
for (size_t i = 0; i < servers_.size(); ++i) {
prepare_server(i, nullptr);
}
}
}
void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout,
td::Promise<td::BufferSlice> promise) override {
QueryInfo query_info = get_query_info(data);
TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info));
send_query_internal(std::move(name), std::move(data), std::move(query_info), server_idx, timeout,
std::move(promise));
}
void send_query_to_server(std::string name, td::BufferSlice data, size_t server_idx, td::Timestamp timeout,
td::Promise<td::BufferSlice> promise) override {
if (server_idx >= servers_.size()) {
promise.set_error(td::Status::Error(PSTRING() << "server idx " << server_idx << " is too big"));
return;
}
server_idx = server_indices_[server_idx];
QueryInfo query_info = get_query_info(data);
prepare_server(server_idx, &query_info);
send_query_internal(std::move(name), std::move(data), std::move(query_info), server_idx, timeout,
std::move(promise));
}
void get_servers_status(td::Promise<std::vector<bool>> promise) override {
std::vector<bool> status(servers_.size());
for (const Server& s : servers_) {
status[s.idx] = s.alive;
}
promise.set_result(std::move(status));
}
void reset_servers() override {
LOG(INFO) << "Force resetting all liteservers";
for (Server& server : servers_) {
server.alive = false;
server.timeout = {};
server.ignore_until = {};
server.client.reset();
}
}
private:
void send_query_internal(std::string name, td::BufferSlice data, QueryInfo query_info, size_t server_idx,
td::Timestamp timeout, td::Promise<td::BufferSlice> promise) {
auto& server = servers_[server_idx];
CHECK(!server.client.empty());
if (!connect_to_all_) {
alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT));
}
td::Promise<td::BufferSlice> P = [SelfId = actor_id(this), server_idx,
promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
if (R.is_error() &&
(R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) {
td::actor::send_closure(SelfId, &ExtClientImpl::on_server_error, server_idx);
}
promise.set_result(std::move(R));
};
LOG(DEBUG) << "Sending query " << query_info.to_str() << " to server #" << server.idx << " ("
<< server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")";
send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout,
std::move(P));
}
td::Result<size_t> select_server(const QueryInfo& query_info) {
for (size_t i = 0; i < servers_.size(); ++i) {
if (servers_[i].alive && servers_[i].config.accepts_query(query_info)) {
return i;
}
}
size_t server_idx = servers_.size();
int cnt = 0;
int best_priority = -1;
for (size_t i = 0; i < servers_.size(); ++i) {
Server& server = servers_[i];
if (!server.config.accepts_query(query_info)) {
continue;
}
int priority = 0;
priority += (server.ignore_until && !server.ignore_until.is_in_past() ? 0 : 10);
if (priority < best_priority) {
continue;
}
if (priority > best_priority) {
best_priority = priority;
cnt = 0;
}
if (td::Random::fast(0, cnt) == 0) {
server_idx = i;
}
++cnt;
}
if (server_idx == servers_.size()) {
return td::Status::Error(PSTRING() << "no liteserver for query " << query_info.to_str());
}
prepare_server(server_idx, &query_info);
return server_idx;
}
void prepare_server(size_t server_idx, const QueryInfo* query_info) {
Server& server = servers_[server_idx];
if (server.alive) {
return;
}
server.alive = true;
server.ignore_until = {};
if (!connect_to_all_) {
alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT));
}
if (!server.client.empty()) {
return;
}
class Callback : public ton::adnl::AdnlExtClient::Callback {
public:
explicit Callback(td::actor::ActorId<ExtClientImpl> parent, size_t idx) : parent_(std::move(parent)), idx_(idx) {
}
void on_ready() override {
}
void on_stop_ready() override {
td::actor::send_closure(parent_, &ExtClientImpl::on_server_error, idx_);
}
private:
td::actor::ActorId<ExtClientImpl> parent_;
size_t idx_;
};
LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":"
<< server.config.addr.get_port() << ") for query " << (query_info ? query_info->to_str() : "[none]");
server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr,
std::make_unique<Callback>(actor_id(this), server_idx));
}
struct Server {
LiteServerConfig config;
size_t idx = 0;
td::actor::ActorOwn<ton::adnl::AdnlExtClient> client;
bool alive = false;
td::Timestamp timeout = td::Timestamp::never();
td::Timestamp ignore_until = td::Timestamp::never();
};
std::vector<Server> servers_;
std::vector<size_t> server_indices_;
td::unique_ptr<Callback> callback_;
bool connect_to_all_ = false;
static constexpr double MAX_NO_QUERIES_TIMEOUT = 100.0;
static constexpr double BAD_SERVER_TIMEOUT = 30.0;
void alarm() override {
if (connect_to_all_) {
return;
}
for (Server& server : servers_) {
if (server.timeout && server.timeout.is_in_past()) {
LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str()
<< ":" << server.config.addr.get_port() << ")";
server.client.reset();
server.alive = false;
server.ignore_until = {};
}
}
}
void on_server_error(size_t idx) {
servers_[idx].alive = false;
servers_[idx].ignore_until = td::Timestamp::in(BAD_SERVER_TIMEOUT);
}
};
td::actor::ActorOwn<ExtClient> ExtClient::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr,
td::unique_ptr<Callback> callback) {
return create({LiteServerConfig{dst, dst_addr}}, std::move(callback));
}
td::actor::ActorOwn<ExtClient> ExtClient::create(std::vector<LiteServerConfig> liteservers,
td::unique_ptr<Callback> callback, bool connect_to_all) {
return td::actor::create_actor<ExtClientImpl>("ExtClient", std::move(liteservers), std::move(callback),
connect_to_all);
}
} // namespace liteclient

48
lite-client/ext-client.h Normal file
View file

@ -0,0 +1,48 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "td/actor/actor.h"
#include "ton/ton-types.h"
#include "adnl/adnl-ext-client.h"
#include "query-utils.hpp"
namespace liteclient {
class ExtClient : public td::actor::Actor {
public:
class Callback {
public:
virtual ~Callback() = default;
};
virtual void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout,
td::Promise<td::BufferSlice> promise) = 0;
virtual void send_query_to_server(std::string name, td::BufferSlice data, size_t server_idx, td::Timestamp timeout,
td::Promise<td::BufferSlice> promise) {
promise.set_error(td::Status::Error("not supported"));
}
virtual void get_servers_status(td::Promise<std::vector<bool>> promise) {
promise.set_error(td::Status::Error("not supported"));
}
virtual void reset_servers() {
}
static td::actor::ActorOwn<ExtClient> create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr,
td::unique_ptr<Callback> callback);
static td::actor::ActorOwn<ExtClient> create(std::vector<LiteServerConfig> liteservers,
td::unique_ptr<Callback> callback, bool connect_to_all = false);
};
} // namespace liteclient

View file

@ -29,22 +29,16 @@
#include "lite-client-common.h"
#include "adnl/adnl-ext-client.h"
#include "tl-utils/lite-utils.hpp"
#include "auto/tl/ton_api_json.h"
#include "auto/tl/lite_api.hpp"
#include "td/utils/OptionParser.h"
#include "td/utils/Time.h"
#include "td/utils/filesystem.h"
#include "td/utils/format.h"
#include "td/utils/Random.h"
#include "td/utils/crypto.h"
#include "td/utils/overloaded.h"
#include "td/utils/port/signals.h"
#include "td/utils/port/stacktrace.h"
#include "td/utils/port/StdStreams.h"
#include "td/utils/port/FileFd.h"
#include "terminal/terminal.h"
#include "ton/lite-tl.hpp"
#include "block/block-db.h"
#include "block/block.h"
@ -58,18 +52,14 @@
#include "vm/vm.h"
#include "vm/cp0.h"
#include "vm/memo.h"
#include "ton/ton-shard.h"
#include "openssl/rand.hpp"
#include "crypto/vm/utils.h"
#include "crypto/common/util.h"
#include "common/checksum.h"
#if TD_DARWIN || TD_LINUX
#include <unistd.h>
#include <fcntl.h>
#endif
#include <iostream>
#include <sstream>
#include "git.h"
using namespace std::literals::string_literals;
@ -77,24 +67,6 @@ using td::Ref;
int verbosity;
std::unique_ptr<ton::adnl::AdnlExtClient::Callback> TestNode::make_callback() {
class Callback : public ton::adnl::AdnlExtClient::Callback {
public:
void on_ready() override {
td::actor::send_closure(id_, &TestNode::conn_ready);
}
void on_stop_ready() override {
td::actor::send_closure(id_, &TestNode::conn_closed);
}
Callback(td::actor::ActorId<TestNode> id) : id_(std::move(id)) {
}
private:
td::actor::ActorId<TestNode> id_;
};
return std::make_unique<Callback>(actor_id(this));
}
void TestNode::run() {
class Cb : public td::TerminalIO::Callback {
public:
@ -110,19 +82,20 @@ void TestNode::run() {
io_ = td::TerminalIO::create("> ", readline_enabled_, ex_mode_, std::make_unique<Cb>(actor_id(this)));
td::actor::send_closure(io_, &td::TerminalIO::set_log_interface);
if (remote_public_key_.empty()) {
std::vector<liteclient::LiteServerConfig> servers;
if (!single_remote_public_key_.empty()) { // Use single provided liteserver
servers.push_back(
liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{single_remote_public_key_}, single_remote_addr_});
td::TerminalIO::out() << "using liteserver " << single_remote_addr_ << "\n";
} else {
auto G = td::read_file(global_config_).move_as_ok();
auto gc_j = td::json_decode(G.as_slice()).move_as_ok();
ton::ton_api::liteclient_config_global gc;
ton::ton_api::from_json(gc, gc_j.get_object()).ensure();
CHECK(gc.liteservers_.size() > 0);
auto idx = liteserver_idx_ >= 0 ? liteserver_idx_
: td::Random::fast(0, static_cast<td::uint32>(gc.liteservers_.size() - 1));
CHECK(idx >= 0 && static_cast<td::uint32>(idx) <= gc.liteservers_.size());
auto& cli = gc.liteservers_[idx];
remote_addr_.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure();
remote_public_key_ = ton::PublicKey{cli->id_};
td::TerminalIO::out() << "using liteserver " << idx << " with addr " << remote_addr_ << "\n";
auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc);
r_servers.ensure();
servers = r_servers.move_as_ok();
if (gc.validator_ && gc.validator_->zero_state_) {
zstate_id_.workchain = gc.validator_->zero_state_->workchain_;
if (zstate_id_.workchain != ton::workchainInvalid) {
@ -131,10 +104,19 @@ void TestNode::run() {
td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n";
}
}
}
client_ =
ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_, make_callback());
if (single_liteserver_idx_ != -1) { // Use single liteserver from config
CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < servers.size());
td::TerminalIO::out() << "using liteserver #" << single_liteserver_idx_ << " with addr "
<< servers[single_liteserver_idx_].addr << "\n";
servers = {servers[single_liteserver_idx_]};
}
}
CHECK(!servers.empty());
client_ = liteclient::ExtClient::create(std::move(servers), nullptr);
ready_ = true;
run_init_queries();
}
void TestNode::got_result(td::Result<td::BufferSlice> R, td::Promise<td::BufferSlice> promise) {
@ -191,8 +173,8 @@ bool TestNode::envelope_send_query(td::BufferSlice query, td::Promise<td::Buffer
});
td::BufferSlice b =
ton::serialize_tl_object(ton::create_tl_object<ton::lite_api::liteServer_query>(std::move(query)), true);
td::actor::send_closure(client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b),
td::Timestamp::in(10.0), std::move(P));
td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", std::move(b), td::Timestamp::in(10.0),
std::move(P));
return true;
}
@ -319,9 +301,10 @@ bool TestNode::get_server_time() {
if (F.is_error()) {
LOG(ERROR) << "cannot parse answer to liteServer.getTime";
} else {
server_time_ = F.move_as_ok()->now_;
server_time_got_at_ = now();
LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")";
mc_server_time_ = F.move_as_ok()->now_;
mc_server_time_got_at_ = now();
LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_
<< ")";
}
}
});
@ -335,7 +318,7 @@ bool TestNode::get_server_version(int mode) {
};
void TestNode::got_server_version(td::Result<td::BufferSlice> res, int mode) {
server_ok_ = false;
mc_server_ok_ = false;
if (res.is_error()) {
LOG(ERROR) << "cannot get server version and time (server too old?)";
} else {
@ -344,11 +327,11 @@ void TestNode::got_server_version(td::Result<td::BufferSlice> res, int mode) {
LOG(ERROR) << "cannot parse answer to liteServer.getVersion";
} else {
auto a = F.move_as_ok();
set_server_version(a->version_, a->capabilities_);
set_server_time(a->now_);
set_mc_server_version(a->version_, a->capabilities_);
set_mc_server_time(a->now_);
}
}
if (!server_ok_) {
if (!mc_server_ok_) {
LOG(ERROR) << "server version is too old (at least " << (min_ls_version >> 8) << "." << (min_ls_version & 0xff)
<< " with capabilities " << min_ls_capabilities << " required), some queries are unavailable";
}
@ -357,24 +340,24 @@ void TestNode::got_server_version(td::Result<td::BufferSlice> res, int mode) {
}
}
void TestNode::set_server_version(td::int32 version, td::int64 capabilities) {
if (server_version_ != version || server_capabilities_ != capabilities) {
server_version_ = version;
server_capabilities_ = capabilities;
LOG(WARNING) << "server version is " << (server_version_ >> 8) << "." << (server_version_ & 0xff)
<< ", capabilities " << server_capabilities_;
void TestNode::set_mc_server_version(td::int32 version, td::int64 capabilities) {
if (mc_server_version_ != version || mc_server_capabilities_ != capabilities) {
mc_server_version_ = version;
mc_server_capabilities_ = capabilities;
LOG(WARNING) << "server version is " << (mc_server_version_ >> 8) << "." << (mc_server_version_ & 0xff)
<< ", capabilities " << mc_server_capabilities_;
}
server_ok_ = (server_version_ >= min_ls_version) && !(~server_capabilities_ & min_ls_capabilities);
mc_server_ok_ = (mc_server_version_ >= min_ls_version) && !(~mc_server_capabilities_ & min_ls_capabilities);
}
void TestNode::set_server_time(int server_utime) {
server_time_ = server_utime;
server_time_got_at_ = now();
LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")";
void TestNode::set_mc_server_time(int server_utime) {
mc_server_time_ = server_utime;
mc_server_time_got_at_ = now();
LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ << ")";
}
bool TestNode::get_server_mc_block_id() {
int mode = (server_capabilities_ & 2) ? 0 : -1;
int mode = (mc_server_capabilities_ & 2) ? 0 : -1;
if (mode < 0) {
auto b = ton::serialize_tl_object(ton::create_tl_object<ton::lite_api::liteServer_getMasterchainInfo>(), true);
return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result<td::BufferSlice> res) -> void {
@ -448,8 +431,8 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt
void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version,
long long capabilities, int last_utime, int server_now) {
set_server_version(version, capabilities);
set_server_time(server_now);
set_mc_server_version(version, capabilities);
set_mc_server_time(server_now);
if (last_utime > server_now) {
LOG(WARNING) << "server claims to have a masterchain block " << blkid.to_str() << " created at " << last_utime
<< " (" << last_utime - server_now << " seconds in the future)";
@ -457,10 +440,10 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI
LOG(WARNING) << "server appears to be out of sync: its newest masterchain block is " << blkid.to_str()
<< " created at " << last_utime << " (" << server_now - last_utime
<< " seconds ago according to the server's clock)";
} else if (last_utime < server_time_got_at_ - 60) {
} else if (last_utime < mc_server_time_got_at_ - 60) {
LOG(WARNING) << "either the server is out of sync, or the local clock is set incorrectly: the newest masterchain "
"block known to server is "
<< blkid.to_str() << " created at " << last_utime << " (" << server_now - server_time_got_at_
<< blkid.to_str() << " created at " << last_utime << " (" << server_now - mc_server_time_got_at_
<< " seconds ago according to the local clock)";
}
got_server_mc_block_id(blkid, zstateid, last_utime);

View file

@ -26,6 +26,7 @@
Copyright 2017-2020 Telegram Systems LLP
*/
#pragma once
#include "ext-client.h"
#include "adnl/adnl-ext-client.h"
#include "tl-utils/tl-utils.hpp"
#include "ton/ton-types.h"
@ -46,22 +47,24 @@ class TestNode : public td::actor::Actor {
min_ls_version = 0x101,
min_ls_capabilities = 1
}; // server version >= 1.1, capabilities at least +1 = build proof chains
td::actor::ActorOwn<ton::adnl::AdnlExtClient> client_;
td::actor::ActorOwn<liteclient::ExtClient> client_;
td::actor::ActorOwn<td::TerminalIO> io_;
bool ready_ = false;
td::int32 single_liteserver_idx_ = -1;
td::IPAddress single_remote_addr_;
ton::PublicKey single_remote_public_key_;
bool readline_enabled_ = true;
bool server_ok_ = false;
td::int32 liteserver_idx_ = -1;
int print_limit_ = 1024;
bool ready_ = false;
bool inited_ = false;
std::string db_root_;
int server_time_ = 0;
int server_time_got_at_ = 0;
int server_version_ = 0;
long long server_capabilities_ = 0;
int mc_server_time_ = 0;
int mc_server_time_got_at_ = 0;
int mc_server_version_ = 0;
long long mc_server_capabilities_ = 0;
bool mc_server_ok_ = false;
ton::ZeroStateIdExt zstate_id_;
ton::BlockIdExt mc_last_id_;
@ -76,9 +79,6 @@ class TestNode : public td::actor::Actor {
const char *parse_ptr_, *parse_end_;
td::Status error_;
td::IPAddress remote_addr_;
ton::PublicKey remote_public_key_;
std::vector<ton::BlockIdExt> known_blk_ids_;
std::size_t shown_blk_ids_ = 0;
@ -89,8 +89,6 @@ class TestNode : public td::actor::Actor {
std::map<td::Bits256, Ref<vm::Cell>> cell_cache_;
std::unique_ptr<ton::adnl::AdnlExtClient::Callback> make_callback();
using creator_stats_func_t =
std::function<bool(const td::Bits256&, const block::DiscountedCounter&, const block::DiscountedCounter&)>;
@ -183,8 +181,8 @@ class TestNode : public td::actor::Actor {
void got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int created_at);
void got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version,
long long capabilities, int last_utime, int server_now);
void set_server_version(td::int32 version, td::int64 capabilities);
void set_server_time(int server_utime);
void set_mc_server_version(td::int32 version, td::int64 capabilities);
void set_mc_server_time(int server_utime);
bool request_block(ton::BlockIdExt blkid);
bool request_state(ton::BlockIdExt blkid);
void got_mc_block(ton::BlockIdExt blkid, td::BufferSlice data);
@ -370,9 +368,6 @@ class TestNode : public td::actor::Actor {
bool parse_shard_id(ton::ShardIdFull& shard);
bool parse_block_id_ext(ton::BlockIdExt& blkid, bool allow_incomplete = false);
bool parse_block_id_ext(std::string blk_id_string, ton::BlockIdExt& blkid, bool allow_incomplete = false) const;
bool parse_stack_value(td::Slice str, vm::StackEntry& value);
bool parse_stack_value(vm::StackEntry& value);
bool parse_stack_values(std::vector<vm::StackEntry>& values);
bool register_blkid(const ton::BlockIdExt& blkid);
bool show_new_blkids(bool all = false);
bool complete_blkid(ton::BlockId partial_blkid, ton::BlockIdExt& complete_blkid) const;
@ -391,16 +386,6 @@ class TestNode : public td::actor::Actor {
static const tlb::TypenameLookup& get_tlb_dict();
public:
void conn_ready() {
LOG(ERROR) << "conn ready";
ready_ = true;
if (!inited_) {
run_init_queries();
}
}
void conn_closed() {
ready_ = false;
}
void set_global_config(std::string str) {
global_config_ = str;
}
@ -411,10 +396,10 @@ class TestNode : public td::actor::Actor {
readline_enabled_ = value;
}
void set_liteserver_idx(td::int32 idx) {
liteserver_idx_ = idx;
single_liteserver_idx_ = idx;
}
void set_remote_addr(td::IPAddress addr) {
remote_addr_ = addr;
single_remote_addr_ = addr;
}
void set_public_key(td::BufferSlice file_name) {
auto R = [&]() -> td::Result<ton::PublicKey> {
@ -425,7 +410,7 @@ class TestNode : public td::actor::Actor {
if (R.is_error()) {
LOG(FATAL) << "bad server public key: " << R.move_as_error();
}
remote_public_key_ = R.move_as_ok();
single_remote_public_key_ = R.move_as_ok();
}
void decode_public_key(td::BufferSlice b64_key) {
auto R = [&]() -> td::Result<ton::PublicKey> {
@ -437,7 +422,7 @@ class TestNode : public td::actor::Actor {
if (R.is_error()) {
LOG(FATAL) << "bad b64 server public key: " << R.move_as_error();
}
remote_public_key_ = R.move_as_ok();
single_remote_public_key_ = R.move_as_ok();
}
void set_fail_timeout(td::Timestamp ts) {
fail_timeout_ = ts;
@ -475,8 +460,7 @@ class TestNode : public td::actor::Actor {
bool envelope_send_query(td::BufferSlice query, td::Promise<td::BufferSlice> promise);
void parse_line(td::BufferSlice data);
TestNode() {
}
TestNode() = default;
void run();
};

400
lite-client/query-utils.cpp Normal file
View file

@ -0,0 +1,400 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#include "query-utils.hpp"
#include "block-parse.h"
#include "td/utils/overloaded.h"
#include "tl-utils/common-utils.hpp"
#include "block/block-auto.h"
#include "auto/tl/lite_api.hpp"
#include "overlay/overlay-broadcast.hpp"
#include "tl-utils/lite-utils.hpp"
#include "ton/lite-tl.hpp"
#include "ton/ton-shard.h"
#include <ton/ton-tl.hpp>
namespace liteclient {
using namespace ton;
std::string QueryInfo::to_str() const {
td::StringBuilder sb;
sb << "[ " << lite_query_name_by_id(query_id) << " " << shard_id.to_str();
switch (type) {
case t_simple:
break;
case t_seqno:
sb << " seqno=" << value;
break;
case t_utime:
sb << " utime=" << value;
break;
case t_lt:
sb << " lt=" << value;
break;
case t_mc_seqno:
sb << " mc_seqno=" << value;
break;
}
sb << " ]";
return sb.as_cslice().str();
}
QueryInfo get_query_info(td::Slice data) {
auto F = fetch_tl_object<lite_api::liteServer_query>(data, true);
if (F.is_ok()) {
data = F.ok()->data_;
} else {
fetch_tl_prefix<lite_api::liteServer_queryPrefix>(data, true).ignore();
}
fetch_tl_prefix<lite_api::liteServer_waitMasterchainSeqno>(data, true).ignore();
auto Q = fetch_tl_object<lite_api::Function>(data, true);
if (Q.is_error()) {
return {};
}
return get_query_info(*Q.ok());
}
QueryInfo get_query_info(const lite_api::Function& f) {
QueryInfo info;
info.query_id = f.get_id();
auto from_block_id = [&](const tl_object_ptr<lite_api::tonNode_blockIdExt>& id) {
BlockIdExt block_id = create_block_id(id);
info.shard_id = block_id.shard_full();
info.type = QueryInfo::t_seqno;
info.value = block_id.seqno();
};
downcast_call(
const_cast<lite_api::Function&>(f),
td::overloaded([&](const lite_api::liteServer_getTime& q) { /* t_simple */ },
[&](const lite_api::liteServer_getVersion& q) { /* t_simple */ },
[&](const lite_api::liteServer_getMasterchainInfo& q) { /* t_simple */ },
[&](const lite_api::liteServer_getMasterchainInfoExt& q) { /* t_simple */ },
[&](const lite_api::liteServer_getBlock& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_getBlockHeader& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_getState& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_getAccountState& q) {
BlockIdExt block_id = create_block_id(q.id_);
AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_);
info.shard_id = acc_id_prefix.as_leaf_shard();
// See LiteQuery::perform_getAccountState
if (block_id.id.workchain != masterchainId) {
info.type = QueryInfo::t_seqno;
info.value = block_id.seqno();
} else if (block_id.id.seqno != ~0U) {
info.type = QueryInfo::t_mc_seqno;
info.value = block_id.seqno();
} else {
info.type = QueryInfo::t_simple;
}
},
[&](const lite_api::liteServer_getAccountStatePrunned& q) {
BlockIdExt block_id = create_block_id(q.id_);
AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_);
info.shard_id = acc_id_prefix.as_leaf_shard();
// See LiteQuery::perform_getAccountState
if (block_id.id.workchain != masterchainId) {
info.type = QueryInfo::t_seqno;
info.value = block_id.seqno();
} else if (block_id.id.seqno != ~0U) {
info.type = QueryInfo::t_mc_seqno;
info.value = block_id.seqno();
} else {
info.type = QueryInfo::t_simple;
}
},
[&](const lite_api::liteServer_getOneTransaction& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_getTransactions& q) {
AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_);
info.shard_id = acc_id_prefix.as_leaf_shard();
info.type = QueryInfo::t_lt;
info.value = q.lt_;
},
[&](const lite_api::liteServer_sendMessage& q) {
info.type = QueryInfo::t_simple;
auto r_root = vm::std_boc_deserialize(q.body_);
if (r_root.is_error()) {
return;
}
block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info;
if (!tlb::unpack_cell_inexact(r_root.ok(), msg_info)) {
return;
}
auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(msg_info.dest);
if (!dest_prefix.is_valid()) {
return;
}
info.shard_id = dest_prefix.as_leaf_shard();
},
[&](const lite_api::liteServer_getShardInfo& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_getAllShardsInfo& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_lookupBlock& q) {
BlockId block_id = create_block_id_simple(q.id_);
info.shard_id = block_id.shard_full();
// See LiteQuery::perform_lookupBlock
if (q.mode_ & 1) {
info.type = QueryInfo::t_seqno;
info.value = block_id.seqno;
} else if (q.mode_ == 2) {
info.type = QueryInfo::t_lt;
info.value = q.lt_;
} else if (q.mode_ == 4) {
info.type = QueryInfo::t_utime;
info.value = q.utime_;
}
},
[&](const lite_api::liteServer_lookupBlockWithProof& q) {
BlockId block_id = create_block_id_simple(q.id_);
info.shard_id = block_id.shard_full();
// See LiteQuery::perform_lookupBlockWithProof
if (q.mode_ & 1) {
info.type = QueryInfo::t_seqno;
info.value = block_id.seqno;
} else if (q.mode_ == 2) {
info.type = QueryInfo::t_lt;
info.value = q.lt_;
} else if (q.mode_ == 4) {
info.type = QueryInfo::t_utime;
info.value = q.utime_;
}
},
[&](const lite_api::liteServer_listBlockTransactions& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_listBlockTransactionsExt& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_getConfigParams& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_getConfigAll& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_getBlockProof& q) {
info.shard_id = ShardIdFull{masterchainId};
BlockIdExt from = create_block_id(q.known_block_);
BlockIdExt to = create_block_id(q.target_block_);
// See LiteQuery::perform_getBlockProof
if ((q.mode_ & 1) && (q.mode_ & 0x1000)) {
info.type = QueryInfo::t_seqno;
info.value = std::max(from.seqno(), to.seqno());
} else {
info.type = QueryInfo::t_simple;
}
},
[&](const lite_api::liteServer_getValidatorStats& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_runSmcMethod& q) {
BlockIdExt block_id = create_block_id(q.id_);
AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_);
info.shard_id = acc_id_prefix.as_leaf_shard();
// See LiteQuery::perform_getAccountState
if (block_id.id.workchain != masterchainId) {
info.type = QueryInfo::t_seqno;
info.value = block_id.seqno();
} else if (block_id.id.seqno != ~0U) {
info.type = QueryInfo::t_mc_seqno;
info.value = block_id.seqno();
} else {
info.type = QueryInfo::t_simple;
}
},
[&](const lite_api::liteServer_getLibraries& q) { /* t_simple */ },
[&](const lite_api::liteServer_getLibrariesWithProof& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_getShardBlockProof& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_nonfinal_getCandidate& q) { /* t_simple */ },
[&](const lite_api::liteServer_nonfinal_getValidatorGroups& q) { /* t_simple */ },
[&](const lite_api::liteServer_getOutMsgQueueSizes& q) {
// This query is expected to be removed, as it is not fully compatible with separated liteservers
/* t_simple */
},
[&](const lite_api::liteServer_getBlockOutMsgQueueSize& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_getDispatchQueueInfo& q) { from_block_id(q.id_); },
[&](const lite_api::liteServer_getDispatchQueueMessages& q) { from_block_id(q.id_); },
[&](const auto&) { /* t_simple */ }));
if (info.shard_id.workchain == masterchainId) {
info.shard_id.shard = shardIdAll;
}
if (!info.shard_id.is_valid_ext()) {
info.shard_id = ShardIdFull{masterchainId};
info.type = QueryInfo::t_simple;
info.value = 0;
}
return info;
}
bool LiteServerConfig::accepts_query(const QueryInfo& query_info) const {
if (is_full) {
return true;
}
for (const Slice& s : slices) {
if (s.accepts_query(query_info)) {
return true;
}
}
return false;
}
bool LiteServerConfig::Slice::accepts_query(const QueryInfo& query_info) const {
if (unlimited) {
for (const ShardInfo& shard : shards_from) {
if (shard_intersects(shard.shard_id, query_info.shard_id)) {
return true;
}
}
return false;
}
if (!shards_from.empty()) {
bool from_ok = false;
DCHECK(shards_from[0].shard_id.is_masterchain());
for (const ShardInfo& shard : shards_from) {
if (shard_intersects(shard.shard_id, query_info.shard_id)) {
switch (query_info.type) {
case QueryInfo::t_simple:
from_ok = true;
break;
case QueryInfo::t_seqno:
from_ok = shard.seqno <= query_info.value;
break;
case QueryInfo::t_utime:
from_ok = shard.utime <= query_info.value;
break;
case QueryInfo::t_lt:
from_ok = shard.lt <= query_info.value;
break;
case QueryInfo::t_mc_seqno:
from_ok = shards_from[0].seqno <= query_info.value;
break;
}
if (from_ok) {
break;
}
}
}
if (!from_ok) {
return false;
}
}
if (!shards_to.empty()) {
bool to_ok = false;
DCHECK(shards_to[0].shard_id.is_masterchain());
for (const ShardInfo& shard : shards_to) {
if (shard_intersects(shard.shard_id, query_info.shard_id)) {
switch (query_info.type) {
case QueryInfo::t_simple:
break;
case QueryInfo::t_seqno:
to_ok = shard.seqno >= query_info.value;
break;
case QueryInfo::t_utime:
to_ok = shard.utime >= query_info.value;
break;
case QueryInfo::t_lt:
to_ok = shard.lt >= query_info.value;
break;
case QueryInfo::t_mc_seqno:
to_ok = shards_from[0].seqno >= query_info.value;
break;
}
if (to_ok) {
break;
}
}
}
if (!to_ok) {
return false;
}
}
return true;
}
td::Result<std::vector<LiteServerConfig>> LiteServerConfig::parse_global_config(
const ton_api::liteclient_config_global& config) {
std::vector<LiteServerConfig> servers;
for (const auto& f : config.liteservers_) {
LiteServerConfig server;
TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_));
server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}};
server.is_full = true;
servers.push_back(std::move(server));
}
for (const auto& f : config.liteservers_v2_) {
LiteServerConfig server;
TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_));
server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}};
server.is_full = false;
for (const auto& slice_obj : f->slices_) {
Slice slice;
td::Status S = td::Status::OK();
downcast_call(*slice_obj,
td::overloaded(
[&](const ton_api::liteserver_descV2_sliceSimple& s) {
slice.unlimited = true;
slice.shards_from.push_back({ShardIdFull{masterchainId}, 0, 0, 0});
for (const auto& shard_obj : s.shards_) {
ShardIdFull shard_id = create_shard_id(shard_obj);
if (!shard_id.is_valid_ext()) {
S = td::Status::Error(PSTRING() << "invalid shard id " << shard_id.to_str());
break;
}
if (!shard_id.is_masterchain()) {
slice.shards_from.push_back({shard_id, 0, 0, 0});
}
}
},
[&](const ton_api::liteserver_descV2_sliceTimed& s) {
auto parse_shards =
[](const std::vector<tl_object_ptr<ton_api::liteserver_descV2_shardInfo>>& shard_objs,
std::vector<ShardInfo>& shards) -> td::Status {
if (shard_objs.empty()) {
return td::Status::OK();
}
size_t i = 0;
int mc_idx = -1;
for (const auto& shard_obj : shard_objs) {
ShardIdFull shard_id = create_shard_id(shard_obj->shard_id_);
if (!shard_id.is_valid_ext()) {
return td::Status::Error(PSTRING() << "invalid shard id " << shard_id.to_str());
}
if (shard_id.is_masterchain()) {
shard_id = ShardIdFull{masterchainId};
if (mc_idx != -1) {
return td::Status::Error("duplicate masterchain shard in sliceTimed");
}
mc_idx = (int)i;
}
shards.push_back({shard_id, (BlockSeqno)shard_obj->seqno_, (UnixTime)shard_obj->utime_,
(LogicalTime)shard_obj->lt_});
++i;
}
if (mc_idx == -1) {
return td::Status::Error("no masterchain shard in sliceTimed");
}
std::swap(shards[0], shards[mc_idx]);
return td::Status::OK();
};
S = parse_shards(s.shards_from_, slice.shards_from);
if (S.is_ok()) {
S = parse_shards(s.shards_to_, slice.shards_to);
}
if (S.is_ok() && slice.shards_from.empty() && slice.shards_to.empty()) {
S = td::Status::Error("shards_from and shards_to are both empty");
}
}));
TRY_STATUS(std::move(S));
server.slices.push_back(slice);
}
servers.push_back(std::move(server));
}
return servers;
}
} // namespace liteclient

View file

@ -0,0 +1,89 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ton/ton-types.h"
#include "auto/tl/lite_api.h"
#include "td/utils/port/IPAddress.h"
#include "adnl/adnl-node-id.hpp"
namespace liteclient {
struct QueryInfo {
enum Type { t_simple, t_seqno, t_utime, t_lt, t_mc_seqno };
int query_id = 0;
ton::ShardIdFull shard_id{ton::masterchainId};
Type type = t_simple;
td::uint64 value = 0;
/* Query types and examples:
* t_simple - query to the recent blocks in a shard, or general info. value = 0.
* getTime, getMasterchainInfo (shard_id = masterchain)
* sendMessage
* getAccountState, runSmcMethod - when no block is given
* t_seqno - query to block with seqno in a shard. value = seqno.
* lookupBlock by seqno
* getBlock, getBlockHeader
* getAccountState, runSmcMethod - when shard block is given
* t_utime - query to a block with given unixtime in a shard. value = utime.
* lookupBlock by utime
* t_lt - query to a block with given lt in a shard. value = lt.
* lookupBlock by lt
* getTransactions
* t_mc_seqno - query to a block in a shard, masterchain seqno is given. value = mc_seqno.
* getAccountState, runSmcMethod - when mc block is given
*/
std::string to_str() const;
};
QueryInfo get_query_info(td::Slice data);
QueryInfo get_query_info(const ton::lite_api::Function& f);
struct LiteServerConfig {
private:
struct ShardInfo {
ton::ShardIdFull shard_id;
ton::BlockSeqno seqno;
ton::UnixTime utime;
ton::LogicalTime lt;
};
struct Slice {
std::vector<ShardInfo> shards_from, shards_to;
bool unlimited = false;
bool accepts_query(const QueryInfo& query_info) const;
};
bool is_full = false;
std::vector<Slice> slices;
public:
ton::adnl::AdnlNodeIdFull adnl_id;
td::IPAddress addr;
LiteServerConfig() = default;
LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress addr)
: is_full(true), adnl_id(adnl_id), addr(addr) {
}
bool accepts_query(const QueryInfo& query_info) const;
static td::Result<std::vector<LiteServerConfig>> parse_global_config(
const ton::ton_api::liteclient_config_global& config);
};
} // namespace liteclient