Tonlib: change liteservers on query timeout or connection close

adnl/adnl-ext-client.hpp

@@ -80,6 +80,9 @@ class AdnlExtClientImpl : public AdnlExtClient {
     if (!conn_.empty() && conn_.get() == conn) {
       callback_->on_stop_ready();
       conn_ = {};
+      for (auto& q : out_queries_) {
+        td::actor::send_closure(q.second, &AdnlQuery::set_error, td::Status::Error(ErrorCode::cancelled));
+      }
       alarm_timestamp() = next_create_at_;
       try_stop();
     }

adnl/adnl-query.cpp

@@ -25,13 +25,16 @@ namespace ton {
 namespace adnl {
 
 void AdnlQuery::alarm() {
-  promise_.set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout"));
+  set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout"));
-  stop();
 }
 void AdnlQuery::result(td::BufferSlice data) {
   promise_.set_value(std::move(data));
   stop();
 }
+void AdnlQuery::set_error(td::Status error) {
+  promise_.set_error(std::move(error));
+  stop();
+}
 
 AdnlQueryId AdnlQuery::random_query_id() {
   AdnlQueryId q_id;

adnl/adnl-query.h

@@ -48,6 +48,7 @@ class AdnlQuery : public td::actor::Actor {
   }
   void alarm() override;
   void result(td::BufferSlice data);
+  void set_error(td::Status error);
   void start_up() override {
     alarm_timestamp() = timeout_;
   }

rldp-http-proxy/DNSResolver.cpp

@@ -25,6 +25,7 @@
 */
 #include "DNSResolver.h"
 #include "td/utils/overloaded.h"
+#include "common/delay.h"
 
 static const double CACHE_TIMEOUT_HARD = 300.0;
 static const double CACHE_TIMEOUT_SOFT = 270.0;
@@ -33,8 +34,18 @@ DNSResolver::DNSResolver(td::actor::ActorId<TonlibClient> tonlib_client) : tonli
 }
 
 void DNSResolver::start_up() {
+  sync();
+}
+
+void DNSResolver::sync() {
   auto obj = tonlib_api::make_object<tonlib_api::sync>();
-  auto P = td::PromiseCreator::lambda([](td::Result<tonlib_api::object_ptr<tonlib_api::Object>>) {});
+  auto P = td::PromiseCreator::lambda([SelfId =
+                                           actor_id(this)](td::Result<tonlib_api::object_ptr<tonlib_api::Object>> R) {
+    if (R.is_error()) {
+      LOG(WARNING) << "Sync error: " << R.move_as_error();
+      ton::delay_action([SelfId]() { td::actor::send_closure(SelfId, &DNSResolver::sync); }, td::Timestamp::in(5.0));
+    }
+  });
   td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P));
 }
 

rldp-http-proxy/DNSResolver.h

@@ -37,6 +37,7 @@ class DNSResolver : public td::actor::Actor {
   void resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdShort> promise);
 
  private:
+  void sync();
   void save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id);
 
   td::actor::ActorId<TonlibClient> tonlib_client_;

tonlib/tonlib/ExtClient.cpp

@@ -35,7 +35,7 @@ void ExtClient::with_last_config(td::Promise<LastConfigState> promise) {
       self->last_config_queries_.extract(query_id).set_result(std::move(result));
     });
   };
-  if (client_.last_block_actor_.empty()) {
+  if (client_.last_config_actor_.empty()) {
     return P.set_error(TonlibError::NoLiteServers());
   }
   td::actor::send_closure(client_.last_config_actor_, &LastConfig::get_last_config, std::move(P));
@@ -62,10 +62,10 @@ void ExtClient::send_raw_query(td::BufferSlice query, td::Promise<td::BufferSlic
       self->queries_.extract(query_id).set_result(std::move(result));
     });
   };
-  if (client_.andl_ext_client_.empty()) {
+  if (client_.adnl_ext_client_.empty()) {
     return P.set_error(TonlibError::NoLiteServers());
   }
-  td::actor::send_closure(client_.andl_ext_client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(query),
+  td::actor::send_closure(client_.adnl_ext_client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(query),
                           td::Timestamp::in(10.0), std::move(P));
 }
 }  // namespace tonlib

tonlib/tonlib/ExtClient.h

@@ -28,6 +28,7 @@
 #include "td/utils/Container.h"
 #include "td/utils/Random.h"
 
+#include "ExtClientLazy.h"
 #include "TonlibError.h"
 #include "utils.h"
 
@@ -37,7 +38,7 @@ class LastConfig;
 struct LastBlockState;
 struct LastConfigState;
 struct ExtClientRef {
-  td::actor::ActorId<ton::adnl::AdnlExtClient> andl_ext_client_;
+  td::actor::ActorId<ExtClientLazy> adnl_ext_client_;
   td::actor::ActorId<LastBlock> last_block_actor_;
   td::actor::ActorId<LastConfig> last_config_actor_;
 };
@@ -94,6 +95,12 @@ class ExtClient {
         });
   }
 
+  void force_change_liteserver() {
+    if (!client_.adnl_ext_client_.empty()) {
+      td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::force_change_liteserver);
+    }
+  }
+
  private:
   ExtClientRef client_;
   td::Container<td::Promise<td::BufferSlice>> queries_;

tonlib/tonlib/ExtClientLazy.cpp

@@ -18,13 +18,20 @@
 */
 #include "ExtClientLazy.h"
 #include "TonlibError.h"
+#include "td/utils/Random.h"
 namespace tonlib {
 
-class ExtClientLazyImp : public ton::adnl::AdnlExtClient {
+class ExtClientLazyImp : public ExtClientLazy {
  public:
-  ExtClientLazyImp(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr,
+  ExtClientLazyImp(std::vector<std::pair<ton::adnl::AdnlNodeIdFull, td::IPAddress>> servers,
                    td::unique_ptr<ExtClientLazy::Callback> callback)
-      : dst_(std::move(dst)), dst_addr_(std::move(dst_addr)), callback_(std::move(callback)) {
+      : servers_(std::move(servers)), callback_(std::move(callback)) {
+    CHECK(!servers_.empty());
+  }
+
+  void start_up() override {
+    td::Random::Fast rnd;
+    td::random_shuffle(td::as_mutable_span(servers_), rnd);
   }
 
   void check_ready(td::Promise<td::Unit> promise) override {
@@ -41,37 +48,66 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient {
     if (client_.empty()) {
       return promise.set_error(TonlibError::Cancelled());
     }
+    td::Promise<td::BufferSlice> P = [SelfId = actor_id(this), idx = cur_server_idx_,
+                                      promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+      if (R.is_error() &&
+          (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) {
+        td::actor::send_closure(SelfId, &ExtClientLazyImp::set_server_bad, idx, true);
+      }
+      promise.set_result(std::move(R));
+    };
     send_closure(client_, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout,
-                 std::move(promise));
+                 std::move(P));
   }
 
+  void force_change_liteserver() override {
+    if (servers_.size() == 1) {
+      return;
+    }
+    cur_server_bad_ = cur_server_bad_force_ = true;
+  }
+
+ private:
   void before_query() {
     if (is_closing_) {
       return;
     }
-    if (!client_.empty()) {
+    alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT);
-      alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT);
+    if (cur_server_bad_) {
+      ++cur_server_idx_;
+    } else if (!client_.empty()) {
       return;
     }
     class Callback : public ton::adnl::AdnlExtClient::Callback {
      public:
-      explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) {
+      explicit Callback(td::actor::ActorShared<ExtClientLazyImp> parent, size_t idx)
+          : parent_(std::move(parent)), idx_(idx) {
       }
       void on_ready() override {
+        td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, false);
       }
       void on_stop_ready() override {
+        td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, true);
       }
 
      private:
-      td::actor::ActorShared<> parent_;
+      td::actor::ActorShared<ExtClientLazyImp> parent_;
+      size_t idx_;
     };
     ref_cnt_++;
-    client_ = ton::adnl::AdnlExtClient::create(dst_, dst_addr_, std::make_unique<Callback>(td::actor::actor_shared()));
+    cur_server_bad_ = false;
+    cur_server_bad_force_ = false;
+    const auto& s = servers_[cur_server_idx_ % servers_.size()];
+    LOG(INFO) << "Connecting to liteserver " << s.second;
+    client_ = ton::adnl::AdnlExtClient::create(
+        s.first, s.second, std::make_unique<Callback>(td::actor::actor_shared(this), cur_server_idx_));
   }
 
- private:
+  std::vector<std::pair<ton::adnl::AdnlNodeIdFull, td::IPAddress>> servers_;
-  ton::adnl::AdnlNodeIdFull dst_;
+  size_t cur_server_idx_ = 0;
-  td::IPAddress dst_addr_;
+  bool cur_server_bad_ = false;
+  bool cur_server_bad_force_ = false;
+
   td::actor::ActorOwn<ton::adnl::AdnlExtClient> client_;
   td::unique_ptr<ExtClientLazy::Callback> callback_;
   static constexpr double MAX_NO_QUERIES_TIMEOUT = 100;
@@ -79,6 +115,11 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient {
   bool is_closing_{false};
   td::uint32 ref_cnt_{1};
 
+  void set_server_bad(size_t idx, bool bad) {
+    if (idx == cur_server_idx_ && servers_.size() > 1 && !cur_server_bad_force_) {
+      cur_server_bad_ = bad;
+    }
+  }
   void alarm() override {
     client_.reset();
   }
@@ -99,9 +140,13 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient {
   }
 };
 
-td::actor::ActorOwn<ton::adnl::AdnlExtClient> ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst,
+td::actor::ActorOwn<ExtClientLazy> ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr,
-                                                                    td::IPAddress dst_addr,
+                                                         td::unique_ptr<Callback> callback) {
-                                                                    td::unique_ptr<Callback> callback) {
+  return create({std::make_pair(dst, dst_addr)}, std::move(callback));
-  return td::actor::create_actor<ExtClientLazyImp>("ExtClientLazy", dst, dst_addr, std::move(callback));
+}
+
+td::actor::ActorOwn<ExtClientLazy> ExtClientLazy::create(
+    std::vector<std::pair<ton::adnl::AdnlNodeIdFull, td::IPAddress>> servers, td::unique_ptr<Callback> callback) {
+  return td::actor::create_actor<ExtClientLazyImp>("ExtClientLazy", std::move(servers), std::move(callback));
 }
 }  // namespace tonlib

tonlib/tonlib/ExtClientLazy.h

@@ -22,15 +22,20 @@
 #include "adnl/adnl-ext-client.h"
 
 namespace tonlib {
-class ExtClientLazy {
+class ExtClientLazy : public ton::adnl::AdnlExtClient {
  public:
   class Callback {
    public:
     virtual ~Callback() {
     }
   };
-  static td::actor::ActorOwn<ton::adnl::AdnlExtClient> create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr,
+
-                                                              td::unique_ptr<Callback> callback);
+  virtual void force_change_liteserver() = 0;
+
+  static td::actor::ActorOwn<ExtClientLazy> create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr,
+                                                   td::unique_ptr<Callback> callback);
+  static td::actor::ActorOwn<ExtClientLazy> create(
+      std::vector<std::pair<ton::adnl::AdnlNodeIdFull, td::IPAddress>> servers, td::unique_ptr<Callback> callback);
 };
 
 }  // namespace tonlib

tonlib/tonlib/ExtClientOutbound.cpp

@@ -38,6 +38,9 @@ class ExtClientOutboundImp : public ExtClientOutbound {
     callback_->request(query_id, data.as_slice().str());
   }
 
+  void force_change_liteserver() override {
+  }
+
   void on_query_result(td::int64 id, td::Result<td::BufferSlice> r_data, td::Promise<td::Unit> promise) override {
     auto it = queries_.find(id);
     if (it == queries_.end()) {

tonlib/tonlib/ExtClientOutbound.h

@@ -19,10 +19,10 @@
 #pragma once
 #include "td/actor/actor.h"
 
-#include "adnl/adnl-ext-client.h"
+#include "ExtClientLazy.h"
 
 namespace tonlib {
-class ExtClientOutbound : public ton::adnl::AdnlExtClient {
+class ExtClientOutbound : public ExtClientLazy {
  public:
   class Callback {
    public:

tonlib/tonlib/LastBlock.cpp

@@ -374,6 +374,7 @@ void LastBlock::on_sync_error(td::Status status) {
     promise.set_error(status.clone());
   }
   promises_.clear();
+  client_.force_change_liteserver();
 }
 void LastBlock::on_fatal_error(td::Status status) {
   VLOG(last_block) << "sync: fatal error " << status;

tonlib/tonlib/LastConfig.cpp

@@ -141,6 +141,7 @@ void LastConfig::on_error(td::Status status) {
     promise.set_error(status.clone());
   }
   promises_.clear();
+  get_config_state_ = QueryState::Empty;
 }
 
 void LastConfig::tear_down() {

tonlib/tonlib/TonlibClient.cpp

@@ -1649,7 +1649,7 @@ void TonlibClient::hangup() {
 
 ExtClientRef TonlibClient::get_client_ref() {
   ExtClientRef ref;
-  ref.andl_ext_client_ = raw_client_.get();
+  ref.adnl_ext_client_ = raw_client_.get();
   ref.last_block_actor_ = raw_last_block_.get();
   ref.last_config_actor_ = raw_last_config_.get();
 
@@ -1683,10 +1683,10 @@ void TonlibClient::init_ext_client() {
     ext_client_outbound_ = client.get();
     raw_client_ = std::move(client);
   } else {
-    auto lite_clients_size = config_.lite_clients.size();
+    std::vector<std::pair<ton::adnl::AdnlNodeIdFull, td::IPAddress>> servers;
-    CHECK(lite_clients_size != 0);
+    for (const auto& s : config_.lite_clients) {
-    auto lite_client_id = td::Random::fast(0, td::narrow_cast<int>(lite_clients_size) - 1);
+      servers.emplace_back(s.adnl_id, s.address);
-    auto& lite_client = config_.lite_clients[lite_client_id];
+    }
     class Callback : public ExtClientLazy::Callback {
      public:
       explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) {
@@ -1697,8 +1697,7 @@ void TonlibClient::init_ext_client() {
     };
     ext_client_outbound_ = {};
     ref_cnt_++;
-    raw_client_ = ExtClientLazy::create(lite_client.adnl_id, lite_client.address,
+    raw_client_ = ExtClientLazy::create(std::move(servers), td::make_unique<Callback>(td::actor::actor_shared()));
-                                        td::make_unique<Callback>(td::actor::actor_shared()));
   }
 }
 

tonlib/tonlib/TonlibClient.h

@@ -110,7 +110,7 @@ class TonlibClient : public td::actor::Actor {
   vm::Dictionary libraries{256};
 
   // network
-  td::actor::ActorOwn<ton::adnl::AdnlExtClient> raw_client_;
+  td::actor::ActorOwn<ExtClientLazy> raw_client_;
   td::actor::ActorId<ExtClientOutbound> ext_client_outbound_;
   td::actor::ActorOwn<LastBlock> raw_last_block_;
   td::actor::ActorOwn<LastConfig> raw_last_config_;

tonlib/tonlib/tonlib-cli.cpp

@@ -174,7 +174,7 @@ class TonlibCli : public td::actor::Actor {
 
   std::map<std::uint64_t, td::Promise<tonlib_api::object_ptr<tonlib_api::Object>>> query_handlers_;
 
-  td::actor::ActorOwn<ton::adnl::AdnlExtClient> raw_client_;
+  td::actor::ActorOwn<tonlib::ExtClientLazy> raw_client_;
 
   bool is_closing_{false};
   td::uint32 ref_cnt_{1};