From 313d37e134172abe2ed22383757e01b49dc6b5e7 Mon Sep 17 00:00:00 2001 From: tolya-yanot <1449561+tolya-yanot@users.noreply.github.com> Date: Fri, 30 Sep 2022 12:14:14 +0300 Subject: [PATCH 1/5] auto-dns & manual-dns smartcontracts updated to actual DNS standard version by starlightduck --- crypto/smartcont/auto-dns.fif | 4 +-- crypto/smartcont/dns-auto-code.fc | 44 +++++++++++++++++++------- crypto/smartcont/dns-manual-code.fc | 33 +++++++++++++------ crypto/smartcont/manual-dns-manage.fif | 6 ++-- 4 files changed, 61 insertions(+), 26 deletions(-) diff --git a/crypto/smartcont/auto-dns.fif b/crypto/smartcont/auto-dns.fif index 6ef2ef0e..21420d1e 100644 --- a/crypto/smartcont/auto-dns.fif +++ b/crypto/smartcont/auto-dns.fif @@ -43,12 +43,12 @@ $# 4 > need-params <> abort"extra parameters, or no parameters for chosen main o variable Values dictnew Values ! // ( i c -- ) { over 0= abort"category cannot be zero" - idict!+ not abort"duplicate category id" + udict!+ not abort"duplicate category id" Values ! } : register-value { @end? abort"category number expected" @next (number) 1 <> abort"category must be integer" - dup 16 fits not abort"category does not fit into 16 bit integer" + dup 256 fits not abort"category does not fit into 256 bit integer" dup 0= abort"category must be non-zero" } : parse-cat-num { @end? abort"smart contract address expected" diff --git a/crypto/smartcont/dns-auto-code.fc b/crypto/smartcont/dns-auto-code.fc index e548a70a..949f1571 100644 --- a/crypto/smartcont/dns-auto-code.fc +++ b/crypto/smartcont/dns-auto-code.fc @@ -7,8 +7,15 @@ | Author: Oleksandr Murzin (tg: @skydev / em: alexhacker64@gmail.com) | | October 2019 | \------------------------------------------------------------------------/ + Updated to actual DNS standard version by starlightduck in 2022 -} +;;===========================================================================;; +;; Custom ASM instructions ;; +;;===========================================================================;; + +cell udict_get_ref_(cell dict, int key_len, int index) asm(index dict key_len) "DICTUGETOPTREF"; + ;;===========================================================================;; ;; Utility functions ;; ;;===========================================================================;; @@ -19,7 +26,7 @@ [OptRef<1b+1r?>:Hashmap(Time|Hash128)->Slice(DomName)>:gc] [UInt<32b>:stdperiod] [Gram:PPReg] [Gram:PPCell] [Gram:PPBit] [UInt<32b>:lasthousekeeping] - := HashmapE 16 ^DNSRecord + := HashmapE 256 (~~16~~) ^DNSRecord STORED DOMAIN NAME SLICE FORMAT: (#ZeroChars<7b>) (Domain name value) #Zeros allows to simultaneously store, for example, com\0 and com\0google\0 @@ -189,6 +196,7 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s store_data(ctl, domdata, gc, [stdper, ppr, ppc, ppb], nhk, lhk); return send_ok(0); } + var (addr, query_id, op) = query_info; if (op == 0x4344656c) { ;; CDel = destroy smart contract ifnot (domdata.null?()) { ;; domain dictionary not empty, force gc @@ -199,9 +207,12 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s ;; domain dictionary still not empty, error return send_error(0xee74656d); } - var (addr, query_id, op) = query_info; return send_message(addr, 0xef6b6179, query_id, op, 0, 128 + 32); } + if (op == 0x54616b65) { ;; Take = take grams from the contract + var amount = in_msg~load_grams(); + return send_message(addr, 0xef6b6179, query_id, op, amount, 64); + } return send_error(0xffffffff); } @@ -289,7 +300,8 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s if (exp >= n) { ;; entry not expired cell cat_table = val~load_ref(); val.end_parse(); - var (cown, ok) = cat_table.idict_get_ref?(16, -2); + ;; update: category length now u256 instead of i16, owner index is now 0 instead of -2 + var (cown, ok) = cat_table.udict_get_ref?(256, 0); if (ok) { owner_info = cown; } @@ -318,14 +330,15 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s data = in_msg~load_ref(); ;; basic integrity check of (client-provided) dictionary ifnot (data.dict_empty?()) { ;; 1000 gas! - var (oinfo, ok) = data.idict_get_ref?(16, -2); + ;; update: category length now u256 instead of i16, owner index is now 0 instead of -2 + var (oinfo, ok) = data.udict_get_ref?(256, 0); if (ok) { var cs = oinfo.begin_parse(); throw_unless(31, cs.slice_bits() >= 16 + 3 + 8 + 256); throw_unless(31, cs.preload_uint(19) == 0x9fd3 * 8 + 4); } - (_, _, int minok) = data.idict_get_min?(16); - (_, _, int maxok) = data.idict_get_max?(16); + (_, _, int minok) = data.udict_get_min?(256); ;; update: category length now u256 instead of i16 + (_, _, int maxok) = data.udict_get_max?(256); ;; update: category length now u256 instead of i16 throw_unless(31, minok & maxok); } } else { @@ -410,7 +423,8 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s (int bits, int refs) = domain.slice_bits_refs(); throw_if(30, refs | (bits & 7)); ;; malformed input (~ 8n-bit) ifnot (bits) { - return (0, null(), 0, null()); ;; zero-length input + ;; return (0, null(), 0, null()); ;; zero-length input + throw(30); ;; update: throw exception for empty input } int domain_last_byte = domain.slice_last(8).preload_uint(8); @@ -420,7 +434,14 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s bits += 8; } if (bits == 8) { - return (0, null(), 0, null()); ;; zero-length input, but with zero byte + return (0, null(), 8, null()); ;; zero-length input, but with zero byte + ;; update: return 8 as resolved, but with no data + } + int domain_first_byte = domain.preload_uint(8); + if (domain_first_byte == 0) { + ;; update: remove prefix \0 + domain~load_uint(8); + bits -= 8; } var ds = get_data().begin_parse(); (_, cell root) = (ds~load_ref(), ds~load_dict()); @@ -453,10 +474,11 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s (int, cell) dnsresolve(slice domain, int category) method_id { (int exp, cell cat_table, int exact?, slice pfx) = dnsdictlookup(domain, now()); ifnot (exp) { - return (0, null()); + return (exact?, null()); ;; update: reuse exact? to return 8 for \0 } ifnot (exact?) { ;; incomplete subdomain found, must return next resolver (-1) - category = -1; + category = "dns_next_resolver"H; ;; 0x19f02441ee588fdb26ee24b2568dd035c3c9206e11ab979be62e55558a1d17ff + ;; update: next resolver is now sha256("dns_next_resolver") instead of -1 } int pfx_bits = pfx.slice_bits(); @@ -467,7 +489,7 @@ int check_owner(cell cat_table, cell owner_info, int src_wc, int src_addr, int s ifnot (category) { return (pfx_bits, cat_table); ;; return cell with entire dictionary for 0 } else { - cell cat_found = cat_table.idict_get_ref(16, category); + cell cat_found = cat_table.udict_get_ref_(256, category); ;; update: category length now u256 instead of i16 return (pfx_bits, cat_found); } } diff --git a/crypto/smartcont/dns-manual-code.fc b/crypto/smartcont/dns-manual-code.fc index 890d5d86..555e5952 100644 --- a/crypto/smartcont/dns-manual-code.fc +++ b/crypto/smartcont/dns-manual-code.fc @@ -7,12 +7,15 @@ | Author: Oleksandr Murzin (tg: @skydev / em: alexhacker64@gmail.com) | | October 2019 | \------------------------------------------------------------------------/ + Updated to actual DNS standard version by starlightduck in 2022 -} ;;===========================================================================;; ;; Custom ASM instructions ;; ;;===========================================================================;; +cell udict_get_ref_(cell dict, int key_len, int index) asm(index dict key_len) "DICTUGETOPTREF"; + (cell, ()) pfxdict_set_ref(cell dict, int key_len, slice key, cell value) { throw_unless(33, dict~pfxdict_set?(key_len, key, begin_cell().store_maybe_ref(value).end_cell().begin_parse())); return (dict, ()); @@ -67,9 +70,9 @@ Operations (continuation of message): 00 Contract initialization message (only if seqno = 0) (x=-) 11 VSet: set specified value to specified subdomain->category (x=2) - [Int<16b>:category] [Name:subdomain] [Cell<1r>:value] + [UInt<256b>:category] [Name:subdomain] [Cell<1r>:value] 12 VDel: delete specified subdomain->category (x=2) - [Int<16b>:category] [Name:subdomain] + [UInt<256b>:category] [Name:subdomain] 21 DSet: replace entire category dictionary of domain with provided (x=0) [Name:subdomain] [Cell<1r>:new_cat_table] 22 DDel: delete entire category dictionary of specified domain (x=0) @@ -112,7 +115,7 @@ int cat = 0; if (op < 20) { ;; for operations with codes 10..19 category is required - cat = ops~load_int(16); + cat = ops~load_uint(256); ;; update: category length now u256 instead of i16 } slice name = null(); ;; any slice value cell cat_table = null(); @@ -159,13 +162,13 @@ ;; 11 VSet: set specified value to specified subdomain->category if (op == 11) { cell new_value = ops~load_maybe_ref(); - cat_table~idict_set_get_ref(16, cat, new_value); + cat_table~udict_set_get_ref(256, cat, new_value); ;; update: category length now u256 instead of i16 root~pfxdict_set_ref(1023, name, cat_table); return (root, ops); } ;; 12 VDel: delete specified subdomain->category value if (op == 12) { - if (cat_table~idict_delete?(16, cat)) { + if (cat_table~udict_delete?(256, cat)) { ;; update: category length now u256 instead of i16 root~pfxdict_set_ref(1023, name, cat_table); } return (root, ops); @@ -261,7 +264,7 @@ cell process_ops(cell root, slice ops) inline_ref { Data structure: Root cell: [UInt<32b>:seqno] [UInt<256b>:owner_public_key] [OptRef<1b+1r?>:HashmapCatTable>:domains] - := HashmapE 16 ^DNSRecord + := HashmapE 256 (~~16~~) ^DNSRecord STORED DOMAIN NAME SLICE FORMAT: (#ZeroChars<7b>) (Domain name value) #Zeros allows to simultaneously store, for example, com\0 and com\0google\0 @@ -291,7 +294,8 @@ int get_public_key() method_id { (int, cell) dnsresolve(slice subdomain, int category) method_id { int bits = subdomain.slice_bits(); ifnot (bits) { - return (0, null()); ;; zero-length input + ;; return (0, null()); ;; zero-length input + throw(30); ;; update: throw exception for empty input } throw_if(30, bits & 7); ;; malformed input (~ 8n-bit) @@ -302,7 +306,14 @@ int get_public_key() method_id { bits += 8; } if (bits == 8) { - return (0, null()); ;; zero-length input, but with zero byte + return (8, null()); ;; zero-length input, but with zero byte + ;; update: return 8 as resolved, but with no data + } + int name_first_byte = subdomain.preload_uint(8); + if (name_first_byte == 0) { + ;; update: remove prefix \0 + subdomain~load_uint(8); + bits -= 8; } (_, _, _, cell root, _) = load_data(); @@ -332,7 +343,9 @@ int get_public_key() method_id { zeros = - zeros; ifnot (tail.slice_empty?()) { ;; if we have tail then len(pfx) < len(subdomain) - category = -1; ;; incomplete subdomain found, must return next resolver (-1) + ;; incomplete subdomain found, must return next resolver + category = "dns_next_resolver"H; ;; 0x19f02441ee588fdb26ee24b2568dd035c3c9206e11ab979be62e55558a1d17ff + ;; update: next resolver is now sha256("dns_next_resolver") instead of -1 } int pfx_bits = pfx.slice_bits() - 7; cell cat_table = val; @@ -342,7 +355,7 @@ int get_public_key() method_id { if (category == 0) { return (pfx_bits, cat_table); ;; return cell with entire dictionary for 0 } else { - cell cat_found = cat_table.idict_get_ref(16, category); + cell cat_found = cat_table.udict_get_ref_(256, category); ;; update: category length now u256 instead of i16 return (pfx_bits, cat_found); } } diff --git a/crypto/smartcont/manual-dns-manage.fif b/crypto/smartcont/manual-dns-manage.fif index ef18127c..98559da9 100644 --- a/crypto/smartcont/manual-dns-manage.fif +++ b/crypto/smartcont/manual-dns-manage.fif @@ -42,7 +42,7 @@ variable Actions { @end? abort"subdomain name expected" @next dup $len 127 > abort"subdomain name too long" } : parse-domain { @end? abort"category number expected" @next (number) 1 <> abort"category must be integer" - dup 16 fits not abort"category does not fit into 16 bit integer" + dup 256 fits not abort"category does not fit into 256 bit integer" dup 0= abort"category must be non-zero" } : parse-cat-num { @end? abort"`cat` expected" @next "cat" $= not abort"`cat` expected" parse-cat-num @@ -107,11 +107,11 @@ file-base +"-dns" +contractid +".addr" load-address { dup first dup `add eq? { drop 4 untuple -rot - Date: Tue, 4 Oct 2022 11:07:38 +0300 Subject: [PATCH 2/5] Bugfixes in rldp-http-proxy and http parser --- http/http-inbound-connection.cpp | 2 +- http/http-outbound-connection.cpp | 2 +- http/http.cpp | 2 +- rldp-http-proxy/rldp-http-proxy.cpp | 6 ++++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/http/http-inbound-connection.cpp b/http/http-inbound-connection.cpp index 89c9c123..57064ddd 100644 --- a/http/http-inbound-connection.cpp +++ b/http/http-inbound-connection.cpp @@ -79,10 +79,10 @@ td::Status HttpInboundConnection::receive(td::ChainBufferReader &input) { send_client_error(); return td::Status::OK(); } + cur_request_ = R.move_as_ok(); if (exit_loop) { return td::Status::OK(); } - cur_request_ = R.move_as_ok(); } auto payload = cur_request_->create_empty_payload().move_as_ok(); diff --git a/http/http-outbound-connection.cpp b/http/http-outbound-connection.cpp index a98efbc9..63621df8 100644 --- a/http/http-outbound-connection.cpp +++ b/http/http-outbound-connection.cpp @@ -42,10 +42,10 @@ td::Status HttpOutboundConnection::receive(td::ChainBufferReader &input) { answer_error(HttpStatusCode::status_bad_request, "", std::move(promise_)); return td::Status::OK(); } + cur_response_ = R.move_as_ok(); if (exit_loop) { return td::Status::OK(); } - cur_response_ = R.move_as_ok(); } if (cur_response_->code() == 100) { diff --git a/http/http.cpp b/http/http.cpp index cefe1a47..cc1fd889 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -587,7 +587,7 @@ tl_object_ptr HttpPayload::store_tl(size_t max_size) max_size -= s.size(); } obj->data_.truncate(obj->data_.size() - S.size()); - if (chunks_.size() != 0) { + if (chunks_.size() != 0 || !parse_completed()) { return obj; } if (!written_zero_chunk_) { diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index a5faa002..0b679c27 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -117,7 +117,7 @@ class HttpRemote : public td::actor::Actor { } }); td::actor::send_closure(client_, &ton::http::HttpClient::send_request, std::move(request), std::move(payload), - td::Timestamp::in(30.0), std::move(P)); + td::Timestamp::never(), std::move(P)); } else { ton::http::answer_error(ton::http::HttpStatusCode::status_bad_request, "", std::move(promise)); } @@ -801,6 +801,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { , dst_(dst) , request_(std::move(request)) , request_payload_(std::move(request_payload)) + , proto_version_(request_->proto_version()) , promise_(std::move(promise)) , adnl_(adnl) , rldp_(rldp) @@ -836,7 +837,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { void abort_query(td::Status error) { LOG(INFO) << "aborting http over rldp query: " << error; - promise_.set_result(create_error_response(request_->proto_version(), 502, "Bad Gateway")); + promise_.set_result(create_error_response(proto_version_, 502, "Bad Gateway")); stop(); } @@ -848,6 +849,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { std::unique_ptr request_; std::shared_ptr request_payload_; + std::string proto_version_; td::Promise promise_; From 33a079962fca393b06168cc497cf2fcdbf3ca854 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 4 Oct 2022 15:12:13 +0300 Subject: [PATCH 3/5] Tonlib: change liteservers on query timeout or connection close --- adnl/adnl-ext-client.hpp | 3 ++ adnl/adnl-query.cpp | 7 ++- adnl/adnl-query.h | 1 + rldp-http-proxy/DNSResolver.cpp | 13 ++++- rldp-http-proxy/DNSResolver.h | 1 + tonlib/tonlib/ExtClient.cpp | 6 +-- tonlib/tonlib/ExtClient.h | 9 +++- tonlib/tonlib/ExtClientLazy.cpp | 77 +++++++++++++++++++++++------ tonlib/tonlib/ExtClientLazy.h | 11 +++-- tonlib/tonlib/ExtClientOutbound.cpp | 3 ++ tonlib/tonlib/ExtClientOutbound.h | 4 +- tonlib/tonlib/LastBlock.cpp | 1 + tonlib/tonlib/LastConfig.cpp | 1 + tonlib/tonlib/TonlibClient.cpp | 13 +++-- tonlib/tonlib/TonlibClient.h | 2 +- tonlib/tonlib/tonlib-cli.cpp | 2 +- 16 files changed, 117 insertions(+), 37 deletions(-) diff --git a/adnl/adnl-ext-client.hpp b/adnl/adnl-ext-client.hpp index a4df818e..13339725 100644 --- a/adnl/adnl-ext-client.hpp +++ b/adnl/adnl-ext-client.hpp @@ -80,6 +80,9 @@ class AdnlExtClientImpl : public AdnlExtClient { if (!conn_.empty() && conn_.get() == conn) { callback_->on_stop_ready(); conn_ = {}; + for (auto& q : out_queries_) { + td::actor::send_closure(q.second, &AdnlQuery::set_error, td::Status::Error(ErrorCode::cancelled)); + } alarm_timestamp() = next_create_at_; try_stop(); } diff --git a/adnl/adnl-query.cpp b/adnl/adnl-query.cpp index 5bc767d2..e098c134 100644 --- a/adnl/adnl-query.cpp +++ b/adnl/adnl-query.cpp @@ -25,13 +25,16 @@ namespace ton { namespace adnl { void AdnlQuery::alarm() { - promise_.set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout")); - stop(); + set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout")); } void AdnlQuery::result(td::BufferSlice data) { promise_.set_value(std::move(data)); stop(); } +void AdnlQuery::set_error(td::Status error) { + promise_.set_error(std::move(error)); + stop(); +} AdnlQueryId AdnlQuery::random_query_id() { AdnlQueryId q_id; diff --git a/adnl/adnl-query.h b/adnl/adnl-query.h index 6e24a49f..3db8c754 100644 --- a/adnl/adnl-query.h +++ b/adnl/adnl-query.h @@ -48,6 +48,7 @@ class AdnlQuery : public td::actor::Actor { } void alarm() override; void result(td::BufferSlice data); + void set_error(td::Status error); void start_up() override { alarm_timestamp() = timeout_; } diff --git a/rldp-http-proxy/DNSResolver.cpp b/rldp-http-proxy/DNSResolver.cpp index a7519742..1fb197a5 100644 --- a/rldp-http-proxy/DNSResolver.cpp +++ b/rldp-http-proxy/DNSResolver.cpp @@ -25,6 +25,7 @@ */ #include "DNSResolver.h" #include "td/utils/overloaded.h" +#include "common/delay.h" static const double CACHE_TIMEOUT_HARD = 300.0; static const double CACHE_TIMEOUT_SOFT = 270.0; @@ -33,8 +34,18 @@ DNSResolver::DNSResolver(td::actor::ActorId tonlib_client) : tonli } void DNSResolver::start_up() { + sync(); +} + +void DNSResolver::sync() { auto obj = tonlib_api::make_object(); - auto P = td::PromiseCreator::lambda([](td::Result>) {}); + auto P = td::PromiseCreator::lambda([SelfId = + actor_id(this)](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Sync error: " << R.move_as_error(); + ton::delay_action([SelfId]() { td::actor::send_closure(SelfId, &DNSResolver::sync); }, td::Timestamp::in(5.0)); + } + }); td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P)); } diff --git a/rldp-http-proxy/DNSResolver.h b/rldp-http-proxy/DNSResolver.h index f87b5e5a..ba52a67e 100644 --- a/rldp-http-proxy/DNSResolver.h +++ b/rldp-http-proxy/DNSResolver.h @@ -37,6 +37,7 @@ class DNSResolver : public td::actor::Actor { void resolve(std::string host, td::Promise promise); private: + void sync(); void save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id); td::actor::ActorId tonlib_client_; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 8c8b5d07..30a29b59 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -35,7 +35,7 @@ void ExtClient::with_last_config(td::Promise promise) { self->last_config_queries_.extract(query_id).set_result(std::move(result)); }); }; - if (client_.last_block_actor_.empty()) { + if (client_.last_config_actor_.empty()) { return P.set_error(TonlibError::NoLiteServers()); } td::actor::send_closure(client_.last_config_actor_, &LastConfig::get_last_config, std::move(P)); @@ -62,10 +62,10 @@ void ExtClient::send_raw_query(td::BufferSlice query, td::Promisequeries_.extract(query_id).set_result(std::move(result)); }); }; - if (client_.andl_ext_client_.empty()) { + if (client_.adnl_ext_client_.empty()) { return P.set_error(TonlibError::NoLiteServers()); } - td::actor::send_closure(client_.andl_ext_client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(query), + td::actor::send_closure(client_.adnl_ext_client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(query), td::Timestamp::in(10.0), std::move(P)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index 882b4794..707ca74b 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -28,6 +28,7 @@ #include "td/utils/Container.h" #include "td/utils/Random.h" +#include "ExtClientLazy.h" #include "TonlibError.h" #include "utils.h" @@ -37,7 +38,7 @@ class LastConfig; struct LastBlockState; struct LastConfigState; struct ExtClientRef { - td::actor::ActorId andl_ext_client_; + td::actor::ActorId adnl_ext_client_; td::actor::ActorId last_block_actor_; td::actor::ActorId last_config_actor_; }; @@ -94,6 +95,12 @@ class ExtClient { }); } + void force_change_liteserver() { + if (!client_.adnl_ext_client_.empty()) { + td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::force_change_liteserver); + } + } + private: ExtClientRef client_; td::Container> queries_; diff --git a/tonlib/tonlib/ExtClientLazy.cpp b/tonlib/tonlib/ExtClientLazy.cpp index 1ab7a24f..335a0ff9 100644 --- a/tonlib/tonlib/ExtClientLazy.cpp +++ b/tonlib/tonlib/ExtClientLazy.cpp @@ -18,13 +18,20 @@ */ #include "ExtClientLazy.h" #include "TonlibError.h" +#include "td/utils/Random.h" namespace tonlib { -class ExtClientLazyImp : public ton::adnl::AdnlExtClient { +class ExtClientLazyImp : public ExtClientLazy { public: - ExtClientLazyImp(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + ExtClientLazyImp(std::vector> servers, td::unique_ptr callback) - : dst_(std::move(dst)), dst_addr_(std::move(dst_addr)), callback_(std::move(callback)) { + : servers_(std::move(servers)), callback_(std::move(callback)) { + CHECK(!servers_.empty()); + } + + void start_up() override { + td::Random::Fast rnd; + td::random_shuffle(td::as_mutable_span(servers_), rnd); } void check_ready(td::Promise promise) override { @@ -41,37 +48,66 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient { if (client_.empty()) { return promise.set_error(TonlibError::Cancelled()); } + td::Promise P = [SelfId = actor_id(this), idx = cur_server_idx_, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error() && + (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { + td::actor::send_closure(SelfId, &ExtClientLazyImp::set_server_bad, idx, true); + } + promise.set_result(std::move(R)); + }; send_closure(client_, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, - std::move(promise)); + std::move(P)); } + void force_change_liteserver() override { + if (servers_.size() == 1) { + return; + } + cur_server_bad_ = cur_server_bad_force_ = true; + } + + private: void before_query() { if (is_closing_) { return; } - if (!client_.empty()) { - alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); + alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); + if (cur_server_bad_) { + ++cur_server_idx_; + } else if (!client_.empty()) { return; } class Callback : public ton::adnl::AdnlExtClient::Callback { public: - explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { + explicit Callback(td::actor::ActorShared parent, size_t idx) + : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { + td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, false); } void on_stop_ready() override { + td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, true); } private: - td::actor::ActorShared<> parent_; + td::actor::ActorShared parent_; + size_t idx_; }; ref_cnt_++; - client_ = ton::adnl::AdnlExtClient::create(dst_, dst_addr_, std::make_unique(td::actor::actor_shared())); + cur_server_bad_ = false; + cur_server_bad_force_ = false; + const auto& s = servers_[cur_server_idx_ % servers_.size()]; + LOG(INFO) << "Connecting to liteserver " << s.second; + client_ = ton::adnl::AdnlExtClient::create( + s.first, s.second, std::make_unique(td::actor::actor_shared(this), cur_server_idx_)); } - private: - ton::adnl::AdnlNodeIdFull dst_; - td::IPAddress dst_addr_; + std::vector> servers_; + size_t cur_server_idx_ = 0; + bool cur_server_bad_ = false; + bool cur_server_bad_force_ = false; + td::actor::ActorOwn client_; td::unique_ptr callback_; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; @@ -79,6 +115,11 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient { bool is_closing_{false}; td::uint32 ref_cnt_{1}; + void set_server_bad(size_t idx, bool bad) { + if (idx == cur_server_idx_ && servers_.size() > 1 && !cur_server_bad_force_) { + cur_server_bad_ = bad; + } + } void alarm() override { client_.reset(); } @@ -99,9 +140,13 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient { } }; -td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, - td::IPAddress dst_addr, - td::unique_ptr callback) { - return td::actor::create_actor("ExtClientLazy", dst, dst_addr, std::move(callback)); +td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback) { + return create({std::make_pair(dst, dst_addr)}, std::move(callback)); +} + +td::actor::ActorOwn ExtClientLazy::create( + std::vector> servers, td::unique_ptr callback) { + return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientLazy.h b/tonlib/tonlib/ExtClientLazy.h index 6014abd3..dc4490b3 100644 --- a/tonlib/tonlib/ExtClientLazy.h +++ b/tonlib/tonlib/ExtClientLazy.h @@ -22,15 +22,20 @@ #include "adnl/adnl-ext-client.h" namespace tonlib { -class ExtClientLazy { +class ExtClientLazy : public ton::adnl::AdnlExtClient { public: class Callback { public: virtual ~Callback() { } }; - static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback); + + virtual void force_change_liteserver() = 0; + + static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback); + static td::actor::ActorOwn create( + std::vector> servers, td::unique_ptr callback); }; } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index fcccc4d4..025ba848 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -38,6 +38,9 @@ class ExtClientOutboundImp : public ExtClientOutbound { callback_->request(query_id, data.as_slice().str()); } + void force_change_liteserver() override { + } + void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) override { auto it = queries_.find(id); if (it == queries_.end()) { diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 7bab4903..87b73b9e 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -19,10 +19,10 @@ #pragma once #include "td/actor/actor.h" -#include "adnl/adnl-ext-client.h" +#include "ExtClientLazy.h" namespace tonlib { -class ExtClientOutbound : public ton::adnl::AdnlExtClient { +class ExtClientOutbound : public ExtClientLazy { public: class Callback { public: diff --git a/tonlib/tonlib/LastBlock.cpp b/tonlib/tonlib/LastBlock.cpp index bc2b74ba..10a4d10f 100644 --- a/tonlib/tonlib/LastBlock.cpp +++ b/tonlib/tonlib/LastBlock.cpp @@ -374,6 +374,7 @@ void LastBlock::on_sync_error(td::Status status) { promise.set_error(status.clone()); } promises_.clear(); + client_.force_change_liteserver(); } void LastBlock::on_fatal_error(td::Status status) { VLOG(last_block) << "sync: fatal error " << status; diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index 0111095b..960d5994 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -141,6 +141,7 @@ void LastConfig::on_error(td::Status status) { promise.set_error(status.clone()); } promises_.clear(); + get_config_state_ = QueryState::Empty; } void LastConfig::tear_down() { diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 842ea5fd..f9e984bb 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1649,7 +1649,7 @@ void TonlibClient::hangup() { ExtClientRef TonlibClient::get_client_ref() { ExtClientRef ref; - ref.andl_ext_client_ = raw_client_.get(); + ref.adnl_ext_client_ = raw_client_.get(); ref.last_block_actor_ = raw_last_block_.get(); ref.last_config_actor_ = raw_last_config_.get(); @@ -1683,10 +1683,10 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - auto lite_clients_size = config_.lite_clients.size(); - CHECK(lite_clients_size != 0); - auto lite_client_id = td::Random::fast(0, td::narrow_cast(lite_clients_size) - 1); - auto& lite_client = config_.lite_clients[lite_client_id]; + std::vector> servers; + for (const auto& s : config_.lite_clients) { + servers.emplace_back(s.adnl_id, s.address); + } class Callback : public ExtClientLazy::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { @@ -1697,8 +1697,7 @@ void TonlibClient::init_ext_client() { }; ext_client_outbound_ = {}; ref_cnt_++; - raw_client_ = ExtClientLazy::create(lite_client.adnl_id, lite_client.address, - td::make_unique(td::actor::actor_shared())); + raw_client_ = ExtClientLazy::create(std::move(servers), td::make_unique(td::actor::actor_shared())); } } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 800d8c80..e7819290 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -110,7 +110,7 @@ class TonlibClient : public td::actor::Actor { vm::Dictionary libraries{256}; // network - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; td::actor::ActorId ext_client_outbound_; td::actor::ActorOwn raw_last_block_; td::actor::ActorOwn raw_last_config_; diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index e889234a..0a042eb1 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -174,7 +174,7 @@ class TonlibCli : public td::actor::Actor { std::map>> query_handlers_; - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; bool is_closing_{false}; td::uint32 ref_cnt_{1}; From 836184e566de6a3e6ee44e0a06f9c5f85b98dfbf Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 4 Oct 2022 18:25:25 +0300 Subject: [PATCH 4/5] Increase maximum size of http request --- rldp-http-proxy/rldp-http-proxy.cpp | 9 ++++----- rldp/rldp-in.hpp | 7 ++++++- rldp/rldp.cpp | 4 ++-- rldp/rldp.h | 6 ++---- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index 0b679c27..a45e6433 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -825,11 +825,9 @@ class RldpToTcpRequestSender : public td::actor::Actor { } void got_result(std::pair, std::shared_ptr> R) { - if (R.first->need_payload()) { - td::actor::create_actor("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_, - rldp_) - .release(); - } + td::actor::create_actor("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_, + rldp_) + .release(); auto f = ton::serialize_tl_object(R.first->store_tl(), true); promise_.set_value(std::move(f)); stop(); @@ -1092,6 +1090,7 @@ class RldpHttpProxy : public td::actor::Actor { } rldp_ = ton::rldp::Rldp::create(adnl_.get()); + td::actor::send_closure(rldp_, &ton::rldp::Rldp::set_default_mtu, 16 << 10); td::actor::send_closure(rldp_, &ton::rldp::Rldp::add_id, local_id_); for (auto &serv_id : server_ids_) { td::actor::send_closure(rldp_, &ton::rldp::Rldp::add_id, serv_id); diff --git a/rldp/rldp-in.hpp b/rldp/rldp-in.hpp index b4981999..9640fd98 100644 --- a/rldp/rldp-in.hpp +++ b/rldp/rldp-in.hpp @@ -71,7 +71,7 @@ class RldpIn : public RldpImpl { void send_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice data) override { - send_query_ex(src, dst, name, std::move(promise), timeout, std::move(data), default_mtu()); + send_query_ex(src, dst, name, std::move(promise), timeout, std::move(data), default_mtu_); } void send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice data, @@ -101,6 +101,10 @@ class RldpIn : public RldpImpl { void add_id(adnl::AdnlNodeIdShort local_id) override; void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) override; + void set_default_mtu(td::uint64 mtu) override { + default_mtu_ = mtu; + } + RldpIn(td::actor::ActorId adnl) : adnl_(adnl) { } @@ -116,6 +120,7 @@ class RldpIn : public RldpImpl { std::set lru_set_; RldpLru lru_; td::uint32 lru_size_ = 0; + td::uint64 default_mtu_ = adnl::Adnl::get_mtu(); std::map max_size_; diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 9b38dcb8..1a772a68 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -116,9 +116,9 @@ void RldpIn::process_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeId } auto ite = max_size_.find(part.transfer_id_); if (ite == max_size_.end()) { - if (static_cast(part.total_size_) > default_mtu()) { + if (static_cast(part.total_size_) > default_mtu_) { VLOG(RLDP_NOTICE) << "dropping too big rldp packet of size=" << part.total_size_ - << " default_mtu=" << default_mtu(); + << " default_mtu=" << default_mtu_; return; } } else { diff --git a/rldp/rldp.h b/rldp/rldp.h index edb19d7a..b8ce3623 100644 --- a/rldp/rldp.h +++ b/rldp/rldp.h @@ -28,15 +28,13 @@ class Rldp : public adnl::AdnlSenderInterface { public: virtual ~Rldp() = default; - static constexpr td::uint64 default_mtu() { - return adnl::Adnl::get_mtu(); - } - virtual void add_id(adnl::AdnlNodeIdShort local_id) = 0; virtual void send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, td::BufferSlice data) = 0; + virtual void set_default_mtu(td::uint64 mtu) = 0; + static td::actor::ActorOwn create(td::actor::ActorId adnl); }; From 5500acd1ba9fcc6782107ff0d10fc69ec48ae2c6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 5 Oct 2022 23:05:23 +0300 Subject: [PATCH 5/5] Minor bugfixes in http --- http/http.cpp | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/http/http.cpp b/http/http.cpp index cc1fd889..dd8ab809 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -279,25 +279,20 @@ td::Status HttpPayload::parse(td::ChainBufferReader &input) { } break; case ParseState::reading_chunk_data: { if (cur_chunk_size_ == 0) { - switch (type_) { - case PayloadType::pt_empty: - UNREACHABLE(); - case PayloadType::pt_eof: - case PayloadType::pt_tunnel: - cur_chunk_size_ = 1LL << 60; - break; - case PayloadType::pt_chunked: - state_ = ParseState::reading_crlf; - break; - case PayloadType::pt_content_length: { - LOG(INFO) << "payload parse success"; - const std::lock_guard lock{mutex_}; - state_ = ParseState::completed; - run_callbacks(); - return td::Status::OK(); - } break; + if (type_ == PayloadType::pt_eof || type_ == PayloadType::pt_tunnel) { + cur_chunk_size_ = 1LL << 60; + } else if (type_ == PayloadType::pt_chunked) { + state_ = ParseState::reading_crlf; + break; + } else if (type_ == PayloadType::pt_content_length) { + LOG(INFO) << "payload parse success"; + const std::lock_guard lock{mutex_}; + state_ = ParseState::completed; + run_callbacks(); + return td::Status::OK(); + } else { + UNREACHABLE(); } - break; } if (input.size() == 0) { return td::Status::OK(); @@ -502,7 +497,7 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt char buf[64]; ::sprintf(buf, "%lx\r\n", s.size()); auto slice = td::Slice(buf, strlen(buf)); - wrote |= !slice.empty(); + wrote = true; output.append(slice); } @@ -514,7 +509,8 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt wrote = true; } } - if (chunks_.size() != 0 || !parse_completed()) { + auto cur_state = state_.load(std::memory_order_consume); + if (chunks_.size() != 0 || (cur_state != ParseState::reading_trailer && cur_state != ParseState::completed)) { return wrote; } if (!written_zero_chunk_) { @@ -531,7 +527,7 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt } while (max_size > 0) { - auto cur_state = state_.load(std::memory_order_consume); + cur_state = state_.load(std::memory_order_consume); HttpHeader h = get_header(); if (h.empty()) { if (cur_state != ParseState::completed) { @@ -587,7 +583,8 @@ tl_object_ptr HttpPayload::store_tl(size_t max_size) max_size -= s.size(); } obj->data_.truncate(obj->data_.size() - S.size()); - if (chunks_.size() != 0 || !parse_completed()) { + auto cur_state = state_.load(std::memory_order_consume); + if (chunks_.size() != 0 || (cur_state != ParseState::reading_trailer && cur_state != ParseState::completed)) { return obj; } if (!written_zero_chunk_) { @@ -597,7 +594,7 @@ tl_object_ptr HttpPayload::store_tl(size_t max_size) LOG(INFO) << "data completed"; while (max_size > 0) { - auto cur_state = state_.load(std::memory_order_consume); + cur_state = state_.load(std::memory_order_consume); HttpHeader h = get_header(); if (h.empty()) { if (cur_state != ParseState::completed) { @@ -869,7 +866,7 @@ td::Status HttpHeader::basic_check() { } for (auto &c : value) { if (c == '\r' || c == '\n') { - return td::Status::Error("bad character in header name"); + return td::Status::Error("bad character in header value"); } } return td::Status::OK();