From 7347ec0b3bbde1d6a7b168dd6382c122bc3a2829 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 29 Dec 2022 17:28:50 +0300 Subject: [PATCH] Add TON Storage to Proxy via storage gateway (#577) * Access ton storage from proxy, resolve "dns_storage_address" in tonlib * Set storage gateway address in proxy args Co-authored-by: SpyCheese --- crypto/block/block.tlb | 2 + crypto/smc-envelope/ManualDns.cpp | 28 ++++++- crypto/smc-envelope/ManualDns.h | 17 ++++- lite-client/lite-client.cpp | 7 ++ rldp-http-proxy/DNSResolver.cpp | 62 +++++++-------- rldp-http-proxy/DNSResolver.h | 6 +- rldp-http-proxy/rldp-http-proxy.cpp | 113 ++++++++++++++++++++-------- tl/generate/scheme/tonlib_api.tl | 1 + tl/generate/scheme/tonlib_api.tlo | Bin 30916 -> 31016 bytes tonlib/tonlib/TonlibClient.cpp | 6 ++ tonlib/tonlib/tonlib-cli.cpp | 2 +- 11 files changed, 171 insertions(+), 73 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 01d18596..902669e8 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -878,6 +878,8 @@ cap_method_pubkey#71f4 = SmcCapability; cap_is_wallet#2177 = SmcCapability; cap_name#ff name:Text = SmcCapability; +dns_storage_address#7473 bag_id:bits256 = DNSRecord; + // // PAYMENT CHANNELS // diff --git a/crypto/smc-envelope/ManualDns.cpp b/crypto/smc-envelope/ManualDns.cpp index 42d379c2..40ce3bac 100644 --- a/crypto/smc-envelope/ManualDns.cpp +++ b/crypto/smc-envelope/ManualDns.cpp @@ -47,6 +47,8 @@ td::StringBuilder& operator<<(td::StringBuilder& sb, const ManualDns::EntryData& .move_as_ok(); case ManualDns::EntryData::Type::SmcAddress: return sb << "SMC:" << data.data.get().smc_address.rserialize(); + case ManualDns::EntryData::Type::StorageAddress: + return sb << "STORAGE:" << data.data.get().bag_id.to_hex(); } return sb << ""; } @@ -93,6 +95,11 @@ td::Result> DnsInterface::EntryData::as_cell() const { smc_address.smc_address.addr); dns.smc_addr = vm::load_cell_slice_ref(cb.finalize()); tlb::pack_cell(res, dns); + }, + [&](const EntryDataStorageAddress& storage_address) { + block::gen::DNSRecord::Record_dns_storage_address dns; + dns.bag_id = storage_address.bag_id; + tlb::pack_cell(res, dns); })); if (error.is_error()) { return error; @@ -142,6 +149,11 @@ td::Result DnsInterface::EntryData::from_cellslice(vm:: } return EntryData::smc_address(block::StdAddress(wc, addr)); } + case block::gen::DNSRecord::dns_storage_address: { + block::gen::DNSRecord::Record_dns_storage_address dns; + tlb::unpack(cs, dns); + return EntryData::storage_address(dns.bag_id); + } } return td::Status::Error("Unknown entry data"); } @@ -536,10 +548,12 @@ std::string DnsInterface::decode_name(td::Slice name) { std::string ManualDns::serialize_data(const EntryData& data) { std::string res; - data.data.visit(td::overloaded([&](const ton::ManualDns::EntryDataText& text) { res = "UNSUPPORTED"; }, - [&](const ton::ManualDns::EntryDataNextResolver& resolver) { res = "UNSUPPORTED"; }, - [&](const ton::ManualDns::EntryDataAdnlAddress& adnl_address) { res = "UNSUPPORTED"; }, - [&](const ton::ManualDns::EntryDataSmcAddress& text) { res = "UNSUPPORTED"; })); + data.data.visit( + td::overloaded([&](const ton::ManualDns::EntryDataText& text) { res = "UNSUPPORTED"; }, + [&](const ton::ManualDns::EntryDataNextResolver& resolver) { res = "UNSUPPORTED"; }, + [&](const ton::ManualDns::EntryDataAdnlAddress& adnl_address) { res = "UNSUPPORTED"; }, + [&](const ton::ManualDns::EntryDataSmcAddress& text) { res = "UNSUPPORTED"; }, + [&](const ton::ManualDns::EntryDataStorageAddress& storage_address) { res = "UNSUPPORTED"; })); return res; } @@ -559,6 +573,12 @@ td::Result> ManualDns::parse_data(td::Slice c } else if (type == "NEXT") { TRY_RESULT(address, block::StdAddress::parse(parser.read_all())); return ManualDns::EntryData::next_resolver(address); + } else if (type == "STORAGE") { + td::Bits256 bag_id; + if (bag_id.from_hex(parser.read_all(), false) != 256) { + return td::Status::Error("failed to parse bag id"); + } + return ManualDns::EntryData::storage_address(bag_id); } else if (parser.data() == "DELETED") { return {}; } diff --git a/crypto/smc-envelope/ManualDns.h b/crypto/smc-envelope/ManualDns.h index 545c3279..b5dee59a 100644 --- a/crypto/smc-envelope/ManualDns.h +++ b/crypto/smc-envelope/ManualDns.h @@ -66,9 +66,19 @@ class DnsInterface { // TODO: capability }; + struct EntryDataStorageAddress { + ton::Bits256 bag_id; + // TODO: proto + bool operator==(const EntryDataStorageAddress& other) const { + return bag_id == other.bag_id; + } + }; + struct EntryData { - enum Type { Empty, Text, NextResolver, AdnlAddress, SmcAddress } type{Empty}; - td::Variant data; + enum Type { Empty, Text, NextResolver, AdnlAddress, SmcAddress, StorageAddress } type{Empty}; + td::Variant + data; static EntryData text(std::string text) { return {Text, EntryDataText{text}}; @@ -82,6 +92,9 @@ class DnsInterface { static EntryData smc_address(block::StdAddress smc_address) { return {SmcAddress, EntryDataSmcAddress{smc_address}}; } + static EntryData storage_address(ton::Bits256 bag_id) { + return {StorageAddress, EntryDataStorageAddress{bag_id}}; + } bool operator==(const EntryData& other) const { return data == other.data; diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index d7d38037..feedbe40 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -1749,6 +1749,13 @@ bool TestNode::show_dns_record(std::ostream& os, td::Bits256 cat, Ref promise) { +void DNSResolver::resolve(std::string host, td::Promise promise) { auto it = cache_.find(host); if (it != cache_.end()) { const CacheEntry &entry = it->second; double now = td::Time::now(); if (now < entry.created_at_ + CACHE_TIMEOUT_HARD) { - promise.set_result(entry.id_); + promise.set_result(entry.address_); promise.reset(); if (now < entry.created_at_ + CACHE_TIMEOUT_SOFT) { return; @@ -73,42 +73,42 @@ void DNSResolver::resolve(std::string host, td::Promiseentries_) { - tonlib_api::downcast_call(*e->entry_.get(), - td::overloaded( - [&](tonlib_api::dns_entryDataAdnlAddress &x) { - if (td::Random::fast(0, cnt) == 0) { - auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_); - if (R.is_ok()) { - id = R.move_as_ok(); - cnt++; - } - } - }, - [&](auto &x) {})); - } - if (cnt == 0) { - if (promise) { - promise.set_error(td::Status::Error("no DNS entries")); - } - } else { - td::actor::send_closure(SelfId, &DNSResolver::save_to_cache, std::move(host), id); - if (promise) { - promise.set_result(id); - } + return; + } + auto obj = R.move_as_ok(); + std::string result; + if (!obj->entries_.empty()) { + tonlib_api::downcast_call(*obj->entries_[0]->entry_, + td::overloaded( + [&](tonlib_api::dns_entryDataAdnlAddress &x) { + auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_); + if (R.is_ok()) { + ton::adnl::AdnlNodeIdShort id = R.move_as_ok(); + result = id.serialize() + ".adnl"; + } + }, + [&](tonlib_api::dns_entryDataStorageAddress &x) { + result = td::to_lower(x.bag_id_.to_hex()) + ".bag"; + }, + [&](auto &x) {})); + } + if (result.empty()) { + if (promise) { + promise.set_error(td::Status::Error("no DNS entries")); } + return; + } + td::actor::send_closure(SelfId, &DNSResolver::save_to_cache, std::move(host), result); + if (promise) { + promise.set_result(std::move(result)); } }); td::actor::send_closure(tonlib_client_, &tonlib::TonlibClientWrapper::send_request, std::move(obj), std::move(P)); } -void DNSResolver::save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id) { +void DNSResolver::save_to_cache(std::string host, std::string address) { CacheEntry &entry = cache_[host]; - entry.id_ = id; + entry.address_ = address; entry.created_at_ = td::Time::now(); } diff --git a/rldp-http-proxy/DNSResolver.h b/rldp-http-proxy/DNSResolver.h index 6b9d31da..6cffe0bd 100644 --- a/rldp-http-proxy/DNSResolver.h +++ b/rldp-http-proxy/DNSResolver.h @@ -34,16 +34,16 @@ class DNSResolver : public td::actor::Actor { explicit DNSResolver(td::actor::ActorId tonlib_client); void start_up() override; - void resolve(std::string host, td::Promise promise); + void resolve(std::string host, td::Promise promise); private: void sync(); - void save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id); + void save_to_cache(std::string host, std::string address); td::actor::ActorId tonlib_client_; struct CacheEntry { - ton::adnl::AdnlNodeIdShort id_; + std::string address_; double created_at_; }; std::map cache_; diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index 9b9e958f..eb04ef9e 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -425,7 +425,8 @@ class TcpToRldpRequestSender : public td::actor::Actor { td::Promise, std::shared_ptr>> promise, td::actor::ActorId adnl, td::actor::ActorId dht, td::actor::ActorId rldp, td::actor::ActorId proxy, - td::actor::ActorId dns_resolver) + td::actor::ActorId dns_resolver, + ton::adnl::AdnlNodeIdShort storage_gateway) : local_id_(local_id) , host_(std::move(host)) , request_(std::move(request)) @@ -435,17 +436,20 @@ class TcpToRldpRequestSender : public td::actor::Actor { , dht_(dht) , rldp_(rldp) , proxy_(proxy) - , dns_resolver_(dns_resolver) { - } - void start_up() override { - resolve(); + , dns_resolver_(dns_resolver) + , storage_gateway_(storage_gateway) { } - void resolve(); + void start_up() override { + td::Random::secure_bytes(id_.as_slice()); + request_tl_ = request_->store_tl(id_); + resolve(host_); + } + + void resolve(std::string host); void resolved(ton::adnl::AdnlNodeIdShort id) { dst_ = id; - td::Random::secure_bytes(id_.as_slice()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -459,7 +463,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { proxy_, is_tunnel()) .release(); - auto f = ton::serialize_tl_object(request_->store_tl(id_), true); + auto f = ton::serialize_tl_object(request_tl_, true); td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, dst_, "http request over rldp", std::move(P), td::Timestamp::in(30.0), std::move(f), 16 << 10); } @@ -538,6 +542,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { std::unique_ptr request_; std::shared_ptr request_payload_; + ton::tl_object_ptr request_tl_; td::Promise, std::shared_ptr>> promise_; @@ -546,6 +551,9 @@ class TcpToRldpRequestSender : public td::actor::Actor { td::actor::ActorId rldp_; td::actor::ActorId proxy_; td::actor::ActorId dns_resolver_; + ton::adnl::AdnlNodeIdShort storage_gateway_ = ton::adnl::AdnlNodeIdShort::zero(); + + bool dns_resolve_sent_ = false; std::unique_ptr response_; std::shared_ptr response_payload_; @@ -983,15 +991,9 @@ class RldpHttpProxy : public td::actor::Actor { } } { - if (is_client_) { - auto D = ton::dht::Dht::create_client(dht_id_, "", dht_config_, keyring_.get(), adnl_.get()); - D.ensure(); - dht_ = D.move_as_ok(); - } else { - auto D = ton::dht::Dht::create(dht_id_, db_root_, dht_config_, keyring_.get(), adnl_.get()); - D.ensure(); - dht_ = D.move_as_ok(); - } + auto D = ton::dht::Dht::create_client(dht_id_, "", dht_config_, keyring_.get(), adnl_.get()); + D.ensure(); + dht_ = D.move_as_ok(); td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_dht_node, dht_.get()); } if (port_) { @@ -1104,15 +1106,20 @@ class RldpHttpProxy : public td::actor::Actor { } } std::transform(host.begin(), host.end(), host.begin(), [](unsigned char c) { return std::tolower(c); }); - if (!proxy_all_ && - (host.size() < 5 || (host.substr(host.size() - 4) != ".ton" && host.substr(host.size() - 5) != ".adnl"))) { + bool allow = proxy_all_; + for (const char* suffix : {".adnl", ".ton", ".bag"}) { + if (td::ends_with(host, td::Slice(suffix))) { + allow = true; + } + } + if (!allow) { promise.set_error(td::Status::Error(ton::ErrorCode::error, "bad server name")); return; } td::actor::create_actor("outboundreq", local_id_, host, std::move(request), std::move(payload), std::move(promise), adnl_.get(), dht_.get(), - rldp_.get(), actor_id(this), dns_resolver_.get()) + rldp_.get(), actor_id(this), dns_resolver_.get(), storage_gateway_) .release(); } @@ -1280,6 +1287,10 @@ class RldpHttpProxy : public td::actor::Actor { proxy_all_ = value; } + void set_storage_gateway(ton::adnl::AdnlNodeIdShort id) { + storage_gateway_ = id; + } + private: struct Host { struct Server { @@ -1317,15 +1328,43 @@ class RldpHttpProxy : public td::actor::Actor { td::actor::ActorOwn tonlib_client_; td::actor::ActorOwn dns_resolver_; + ton::adnl::AdnlNodeIdShort storage_gateway_ = ton::adnl::AdnlNodeIdShort::zero(); std::map, td::Promise)>> payload_senders_; }; -void TcpToRldpRequestSender::resolve() { - auto S = td::Slice(host_); - if (S.size() >= 5 && S.substr(S.size() - 5) == ".adnl") { +void TcpToRldpRequestSender::resolve(std::string host) { + auto S = td::Slice(host); + if (td::ends_with(S, ".bag")) { + if (storage_gateway_.is_zero()) { + abort_query(td::Status::Error("storage gateway is not set")); + return; + } + td::Slice bag_id = S.substr(0, S.size() - 4); + td::Slice url = request_tl_->url_; + if (td::begins_with(url, "http://")) { + url.remove_prefix(7); + } + size_t pos = url.find('/'); + if (pos == td::Slice::npos) { + url = "/"; + } else { + url = url.substr(pos); + } + request_tl_->url_ = (PSTRING() << "/gateway/" << bag_id << url); + host = storage_gateway_.serialize() + ".adnl"; + for (auto& header : request_tl_->headers_) { + if (td::to_lower(header->name_) == "host") { + header->value_ = host; + break; + } + } + resolved(storage_gateway_); + return; + } + if (td::ends_with(S, ".adnl")) { S.truncate(S.size() - 5); auto R = ton::adnl::AdnlNodeIdShort::parse(S); if (R.is_error()) { @@ -1335,15 +1374,20 @@ void TcpToRldpRequestSender::resolve() { resolved(R.move_as_ok()); return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, - R.move_as_error_prefix("failed to resolve: ")); - } else { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolved, R.move_as_ok()); - } - }); - td::actor::send_closure(dns_resolver_, &DNSResolver::resolve, host_, std::move(P)); + if (dns_resolve_sent_) { + abort_query(td::Status::Error(PSTRING() << "unexpected dns result: " << host)); + return; + } + dns_resolve_sent_ = true; + td::actor::send_closure(dns_resolver_, &DNSResolver::resolve, host, + [SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, + R.move_as_error_prefix("failed to resolve: ")); + } else { + td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolve, R.move_as_ok()); + } + }); } void HttpRldpPayloadSender::start_up() { @@ -1532,6 +1576,11 @@ int main(int argc, char *argv[]) { logger_ = td::FileLog::create(fname.str()).move_as_ok(); td::log_interface = logger_.get(); }); + p.add_checked_option('S', "storage-gateway", "adnl address of ton storage gateway", [&](td::Slice arg) -> td::Status { + TRY_RESULT(adnl, ton::adnl::AdnlNodeIdShort::parse(arg)); + td::actor::send_closure(x, &RldpHttpProxy::set_storage_gateway, adnl); + return td::Status::OK(); + }); p.add_checked_option('P', "proxy-all", "value=[YES|NO]. proxy all HTTP requests (default only *.ton and *.adnl)", [&](td::Slice value) { if (value == "YES" || value == "yes") { diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 553ae4f2..10a5896c 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -119,6 +119,7 @@ dns.entryDataText text:string = dns.EntryData; dns.entryDataNextResolver resolver:AccountAddress = dns.EntryData; dns.entryDataSmcAddress smc_address:AccountAddress = dns.EntryData; dns.entryDataAdnlAddress adnl_address:AdnlAddress = dns.EntryData; +dns.entryDataStorageAddress bag_id:int256 = dns.EntryData; dns.entry name:string category:int256 entry:dns.EntryData = dns.Entry; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index db7810e3823909ac183089dfcde1dfca084f890d..075c5a3abdf705679b48dd79a799a1f70a9d67a9 100644 GIT binary patch delta 222 zcmX@|k#WT*#tk~0q9V6B&+?|^73;a?l@wLFB$gyHFtBZQz3q+)W4 znGGk1ar2mv598#*ENzJJ4Ko!mW0JW6M8L#c2dcr`1j3kOt^r}3fQn4AfGRPtuy6;N z$p8gc<+x&GLKaMy2014c=A7V?{G!D4RL7K*qSWGIxEp|aq@Uf_WJ^j+kIzh*oNg)$ LwqV6DaiQZYHj z%m%_Z6J!8k+%QuCGbWis1x(C!plZ!cAdETY8W6?_sK_J>s1gGU3$V=Q3l to_dns_entry_data(tonlib_api::dns_EntryDat TRY_RESULT(address, get_adnl_address(adnl_address.adnl_address_->adnl_address_)); return ton::ManualDns::EntryData::adnl_address(std::move(address)); }, + [&](tonlib_api::dns_entryDataStorageAddress& storage_address) -> R { + return ton::ManualDns::EntryData::storage_address(storage_address.bag_id_); + }, [&](tonlib_api::dns_entryDataText& text) -> R { return ton::ManualDns::EntryData::text(text.text_); })); } @@ -4093,6 +4096,9 @@ td::Result> to_tonlib_api( [&](const ton::ManualDns::EntryDataSmcAddress& smc_address) { res = tonlib_api::make_object( tonlib_api::make_object(smc_address.smc_address.rserialize(true))); + }, + [&](const ton::ManualDns::EntryDataStorageAddress& storage_address) { + res = tonlib_api::make_object(storage_address.bag_id); })); return res; } diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 0a042eb1..db7497b0 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -307,7 +307,7 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << "dns cmdfile \n"; td::TerminalIO::out() << "\t = set | delete.name | delete.all\n"; td::TerminalIO::out() << "\t = DELETED | EMPTY | TEXT: | NEXT: | SMC: | " - "ADNL:\n"; + "ADNL: | STORAGE:\n"; } void pchan_help() {