From 53ec9684bd213983e1fe0f7610d3e3453a4ec628 Mon Sep 17 00:00:00 2001 From: ton Date: Sun, 2 Feb 2020 16:53:37 +0400 Subject: [PATCH] updated liteserver - new methods for liteserver/liteclient - added ADNL/DHT client-only work mode - fixed crash in ADNL --- GPLv2 | 2 +- LGPLv2 | 2 +- adnl/adnl-address-list.cpp | 10 + adnl/adnl-address-list.h | 2 + adnl/adnl-channel.cpp | 8 +- adnl/adnl-channel.h | 2 +- adnl/adnl-channel.hpp | 2 +- adnl/adnl-local-id.cpp | 13 +- adnl/adnl-local-id.h | 2 +- adnl/adnl-network-manager.cpp | 2 +- adnl/adnl-packet.h | 9 + adnl/adnl-peer-table.cpp | 8 +- adnl/adnl-peer-table.h | 2 +- adnl/adnl-peer-table.hpp | 2 +- adnl/adnl-peer.cpp | 16 +- .../blockchain-explorer-http.cpp | 18 +- .../blockchain-explorer-http.hpp | 2 + blockchain-explorer/blockchain-explorer.cpp | 4 + blockchain-explorer/blockchain-explorer.hpp | 2 + crypto/block/check-proof.cpp | 15 +- crypto/block/check-proof.h | 7 +- crypto/block/mc-config.cpp | 10 +- crypto/fift/lib/Asm.fif | 7 +- crypto/fift/words.cpp | 11 +- crypto/smartcont/highload-wallet-code.fc | 6 + crypto/smartcont/highload-wallet-v2-code.fc | 6 + crypto/smartcont/new-wallet-v2.fif | 4 +- crypto/smartcont/new-wallet-v3.fif | 4 +- crypto/smartcont/new-wallet.fif | 4 +- crypto/smartcont/restricted-wallet-code.fc | 6 + crypto/smartcont/restricted-wallet2-code.fc | 6 + crypto/smartcont/simple-wallet-ext-code.fc | 5 + crypto/smartcont/wallet-code.fc | 6 + crypto/smartcont/wallet3-code.fc | 6 + crypto/vm/cells/CellBuilder.cpp | 16 +- crypto/vm/cells/CellSlice.h | 3 +- crypto/vm/dict.cpp | 8 +- crypto/vm/dictops.cpp | 39 ++- crypto/vm/log.h | 8 +- crypto/vm/stack.cpp | 4 +- dht/dht-bucket.cpp | 6 +- dht/dht-bucket.hpp | 7 +- dht/dht-in.hpp | 6 +- dht/dht-query.cpp | 21 +- dht/dht-query.hpp | 27 +- dht/dht-remote-node.cpp | 67 ++-- dht/dht-remote-node.hpp | 3 +- dht/dht.cpp | 135 +++++--- dht/dht.h | 4 + dht/dht.hpp | 4 +- doc/tvm.tex | 5 + keyring/keyring.cpp | 16 +- lite-client/lite-client.cpp | 317 ++++++++++++------ lite-client/lite-client.h | 10 +- tdutils/td/utils/BufferedFd.h | 8 +- tdutils/td/utils/FileLog.cpp | 8 +- tdutils/td/utils/FileLog.h | 4 +- tdutils/td/utils/misc.h | 8 +- tdutils/td/utils/port/detail/Epoll.cpp | 3 +- tdutils/test/misc.cpp | 7 +- test/regression-tests.ans | 2 +- tl/generate/scheme/lite_api.tl | 4 +- tl/generate/scheme/lite_api.tlo | Bin 12368 -> 12416 bytes tl/generate/scheme/ton_api.tl | 22 ++ tl/generate/scheme/ton_api.tlo | Bin 56528 -> 58212 bytes validator/db/archive-manager.cpp | 8 +- validator/db/archive-slice.cpp | 1 + validator/db/package.cpp | 10 +- validator/impl/liteserver.cpp | 120 ++++++- validator/impl/liteserver.hpp | 16 +- 70 files changed, 816 insertions(+), 322 deletions(-) diff --git a/GPLv2 b/GPLv2 index c2a8d683..c1e0c158 100644 --- a/GPLv2 +++ b/GPLv2 @@ -23,5 +23,5 @@ exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. - Copyright 2017-2019 Telegram Systems LLP + Copyright 2017-2020 Telegram Systems LLP */ diff --git a/LGPLv2 b/LGPLv2 index d93ab1b9..ab2bc125 100644 --- a/LGPLv2 +++ b/LGPLv2 @@ -14,5 +14,5 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - Copyright 2017-2019 Telegram Systems LLP + Copyright 2017-2020 Telegram Systems LLP */ diff --git a/adnl/adnl-address-list.cpp b/adnl/adnl-address-list.cpp index ab395144..8b92d925 100644 --- a/adnl/adnl-address-list.cpp +++ b/adnl/adnl-address-list.cpp @@ -155,6 +155,16 @@ td::Result AdnlAddressList::create(const tl_object_ptr(addr.get_ipv4(), static_cast(addr.get_port())); + addrs_.push_back(std::move(r)); + return td::Status::OK(); + } else { + return td::Status::Error(ErrorCode::protoviolation, "only works with ipv4"); + } +} + } // namespace adnl } // namespace ton diff --git a/adnl/adnl-address-list.h b/adnl/adnl-address-list.h index d13fce6f..686ced65 100644 --- a/adnl/adnl-address-list.h +++ b/adnl/adnl-address-list.h @@ -88,6 +88,7 @@ class AdnlAddressList { void add_addr(AdnlAddress addr) { addrs_.push_back(addr); } + void update(td::IPAddress addr); bool public_only() const; td::uint32 size() const { return static_cast(addrs_.size()); @@ -98,6 +99,7 @@ class AdnlAddressList { } static td::Result create(const tl_object_ptr &addr_list); + td::Status add_udp_address(td::IPAddress addr); }; } // namespace adnl diff --git a/adnl/adnl-channel.cpp b/adnl/adnl-channel.cpp index 9314d54e..f4e09cb0 100644 --- a/adnl/adnl-channel.cpp +++ b/adnl/adnl-channel.cpp @@ -111,13 +111,15 @@ void AdnlChannelImpl::send_message(td::uint32 priority, td::actor::ActorId R) { + [peer = peer_pair_, channel_id = channel_in_id_, addr, id = print_id()](td::Result R) { if (R.is_error()) { VLOG(ADNL_WARNING) << id << ": dropping IN message: can not decrypt: " << R.move_as_error(); } else { - td::actor::send_closure(peer, &AdnlPeerPair::receive_packet_from_channel, channel_id, R.move_as_ok()); + auto packet = R.move_as_ok(); + packet.set_remote_addr(addr); + td::actor::send_closure(peer, &AdnlPeerPair::receive_packet_from_channel, channel_id, std::move(packet)); } }); diff --git a/adnl/adnl-channel.h b/adnl/adnl-channel.h index e56f040c..a8e2c448 100644 --- a/adnl/adnl-channel.h +++ b/adnl/adnl-channel.h @@ -35,7 +35,7 @@ class AdnlChannel : public td::actor::Actor { AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, AdnlChannelIdShort &out_id, AdnlChannelIdShort &in_id, td::actor::ActorId peer_pair); - virtual void receive(td::BufferSlice data) = 0; + virtual void receive(td::IPAddress addr, td::BufferSlice data) = 0; virtual void send_message(td::uint32 priority, td::actor::ActorId conn, td::BufferSlice data) = 0; virtual ~AdnlChannel() = default; diff --git a/adnl/adnl-channel.hpp b/adnl/adnl-channel.hpp index 1499f5c4..d4356f93 100644 --- a/adnl/adnl-channel.hpp +++ b/adnl/adnl-channel.hpp @@ -33,7 +33,7 @@ class AdnlChannelImpl : public AdnlChannel { AdnlChannelIdShort in_id, AdnlChannelIdShort out_id, std::unique_ptr encryptor, std::unique_ptr decryptor); void decrypt(td::BufferSlice data, td::Promise promise); - void receive(td::BufferSlice data) override; + void receive(td::IPAddress addr, td::BufferSlice data) override; void send_message(td::uint32 priority, td::actor::ActorId conn, td::BufferSlice data) override; struct AdnlChannelPrintId { diff --git a/adnl/adnl-local-id.cpp b/adnl/adnl-local-id.cpp index 79b2bcd2..cd9d384e 100644 --- a/adnl/adnl-local-id.cpp +++ b/adnl/adnl-local-id.cpp @@ -40,13 +40,15 @@ AdnlAddressList AdnlLocalId::get_addr_list() const { return addr_list_; } -void AdnlLocalId::receive(td::BufferSlice data) { +void AdnlLocalId::receive(td::IPAddress addr, td::BufferSlice data) { auto P = td::PromiseCreator::lambda( - [peer_table = peer_table_, dst = short_id_, id = print_id()](td::Result R) { + [peer_table = peer_table_, dst = short_id_, addr, id = print_id()](td::Result R) { if (R.is_error()) { VLOG(ADNL_WARNING) << id << ": dropping IN message: cannot decrypt: " << R.move_as_error(); } else { - td::actor::send_closure(peer_table, &AdnlPeerTable::receive_decrypted_packet, dst, R.move_as_ok()); + auto packet = R.move_as_ok(); + packet.set_remote_addr(addr); + td::actor::send_closure(peer_table, &AdnlPeerTable::receive_decrypted_packet, dst, std::move(packet)); } }); @@ -117,7 +119,7 @@ void AdnlLocalId::update_address_list(AdnlAddressList addr_list) { } void AdnlLocalId::publish_address_list() { - if (dht_node_.empty() || addr_list_.empty()) { + if (dht_node_.empty() || addr_list_.empty() || addr_list_.size() == 0) { VLOG(ADNL_NOTICE) << this << ": skipping public addr list, because localid (or dht node) not fully initialized"; return; } @@ -178,7 +180,8 @@ AdnlLocalId::AdnlLocalId(AdnlNodeIdFull id, AdnlAddressList addr_list, td::actor id_ = std::move(id); short_id_ = id_.compute_short_id(); addr_list_ = std::move(addr_list); - if (addr_list_.addrs().size() > 0) { + if (!addr_list_.empty()) { + addr_list_.set_reinit_date(Adnl::adnl_start_time()); addr_list_.set_version(static_cast(td::Clocks::system())); } peer_table_ = peer_table; diff --git a/adnl/adnl-local-id.h b/adnl/adnl-local-id.h index 07326f79..03998f7a 100644 --- a/adnl/adnl-local-id.h +++ b/adnl/adnl-local-id.h @@ -54,7 +54,7 @@ class AdnlLocalId : public td::actor::Actor { void decrypt_message(td::BufferSlice data, td::Promise promise); void deliver(AdnlNodeIdShort src, td::BufferSlice data); void deliver_query(AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); - void receive(td::BufferSlice data); + void receive(td::IPAddress addr, td::BufferSlice data); void subscribe(std::string prefix, std::unique_ptr callback); void unsubscribe(std::string prefix); diff --git a/adnl/adnl-network-manager.cpp b/adnl/adnl-network-manager.cpp index c5c25463..02e73ce0 100644 --- a/adnl/adnl-network-manager.cpp +++ b/adnl/adnl-network-manager.cpp @@ -59,7 +59,7 @@ void AdnlNetworkManagerImpl::receive_udp_message(td::UdpMessage message) { } received_messages_++; if (received_messages_ % 64 == 0) { - VLOG(ADNL_DEBUG) << this << ": received " << received_messages_ << "udp messages"; + VLOG(ADNL_DEBUG) << this << ": received " << received_messages_ << " udp messages"; } VLOG(ADNL_EXTRA_DEBUG) << this << ": received message of size " << message.data.size(); diff --git a/adnl/adnl-packet.h b/adnl/adnl-packet.h index 6459e06b..107ed7e5 100644 --- a/adnl/adnl-packet.h +++ b/adnl/adnl-packet.h @@ -132,6 +132,9 @@ class AdnlPacket { auto signature() const { return signature_.clone(); } + auto remote_addr() const { + return remote_addr_; + } void init_random(); @@ -188,6 +191,10 @@ class AdnlPacket { flags_ |= Flags::f_reinit_date; } + void set_remote_addr(td::IPAddress addr) { + remote_addr_ = addr; + } + private: td::BufferSlice rand1_; td::uint32 flags_{0}; @@ -204,6 +211,8 @@ class AdnlPacket { td::int32 dst_reinit_date_{0}; td::BufferSlice signature_; td::BufferSlice rand2_; + + td::IPAddress remote_addr_; }; } // namespace adnl diff --git a/adnl/adnl-peer-table.cpp b/adnl/adnl-peer-table.cpp index 04a77684..b4880de2 100644 --- a/adnl/adnl-peer-table.cpp +++ b/adnl/adnl-peer-table.cpp @@ -49,7 +49,7 @@ td::actor::ActorOwn Adnl::create(std::string db, td::actor::ActorId(td::actor::create_actor("PeerTable", db, keyring)); } -void AdnlPeerTableImpl::receive_packet(td::BufferSlice data) { +void AdnlPeerTableImpl::receive_packet(td::IPAddress addr, td::BufferSlice data) { if (data.size() < 32) { VLOG(ADNL_WARNING) << this << ": dropping IN message [?->?]: message too short: len=" << data.size(); return; @@ -60,14 +60,14 @@ void AdnlPeerTableImpl::receive_packet(td::BufferSlice data) { auto it = local_ids_own_.find(dst); if (it != local_ids_own_.end()) { - td::actor::send_closure(it->second, &AdnlLocalId::receive, std::move(data)); + td::actor::send_closure(it->second, &AdnlLocalId::receive, addr, std::move(data)); return; } AdnlChannelIdShort dst_chan_id{dst.pubkey_hash()}; auto it2 = channels_.find(dst_chan_id); if (it2 != channels_.end()) { - td::actor::send_closure(it2->second, &AdnlChannel::receive, std::move(data)); + td::actor::send_closure(it2->second, &AdnlChannel::receive, addr, std::move(data)); return; } @@ -237,7 +237,7 @@ void AdnlPeerTableImpl::register_network_manager(td::actor::ActorId id) : id_(id) { } diff --git a/adnl/adnl-peer-table.h b/adnl/adnl-peer-table.h index 1050222c..42447a44 100644 --- a/adnl/adnl-peer-table.h +++ b/adnl/adnl-peer-table.h @@ -89,7 +89,7 @@ class AdnlPeerTable : public Adnl { virtual void answer_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlQueryId query_id, td::BufferSlice data) = 0; - virtual void receive_packet(td::BufferSlice data) = 0; + virtual void receive_packet(td::IPAddress addr, td::BufferSlice data) = 0; virtual void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet) = 0; virtual void send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message) = 0; diff --git a/adnl/adnl-peer-table.hpp b/adnl/adnl-peer-table.hpp index a4873727..c5c6acd8 100644 --- a/adnl/adnl-peer-table.hpp +++ b/adnl/adnl-peer-table.hpp @@ -43,7 +43,7 @@ class AdnlPeerTableImpl : public AdnlPeerTable { void add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, AdnlAddressList addr_list) override; void add_static_nodes_from_config(AdnlNodesList nodes) override; - void receive_packet(td::BufferSlice data) override; + void receive_packet(td::IPAddress addr, td::BufferSlice data) override; void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket data) override; void send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message) override; void send_message(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data) override { diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index ba077682..4ff8044a 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -131,7 +131,12 @@ void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) { } if (packet.dst_reinit_date() > 0 && packet.dst_reinit_date() < d) { if (!packet.addr_list().empty()) { - update_addr_list(packet.addr_list()); + auto addr_list = packet.addr_list(); + if (packet.remote_addr().is_valid() && addr_list.size() == 0) { + VLOG(ADNL_DEBUG) << "adding implicit address " << packet.remote_addr(); + addr_list.add_udp_address(packet.remote_addr()); + } + update_addr_list(std::move(addr_list)); } if (!packet.priority_addr_list().empty()) { update_addr_list(packet.priority_addr_list()); @@ -174,7 +179,12 @@ void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) { } if (!packet.addr_list().empty()) { - update_addr_list(packet.addr_list()); + auto addr_list = packet.addr_list(); + if (packet.remote_addr().is_valid() && addr_list.size() == 0) { + VLOG(ADNL_DEBUG) << "adding implicit address " << packet.remote_addr(); + addr_list.add_udp_address(packet.remote_addr()); + } + update_addr_list(std::move(addr_list)); } if (!packet.priority_addr_list().empty()) { update_addr_list(packet.priority_addr_list()); @@ -642,7 +652,7 @@ void AdnlPeerPairImpl::update_addr_list(AdnlAddressList addr_list) { if (addr_list.empty()) { return; } - CHECK(addr_list.size() > 0); + //CHECK(addr_list.size() > 0); if (addr_list.reinit_date() > td::Clocks::system() + 60) { VLOG(ADNL_WARNING) << "dropping addr list with too new reinit date"; diff --git a/blockchain-explorer/blockchain-explorer-http.cpp b/blockchain-explorer/blockchain-explorer-http.cpp index d5d41a15..0d0e0c95 100644 --- a/blockchain-explorer/blockchain-explorer-http.cpp +++ b/blockchain-explorer/blockchain-explorer-http.cpp @@ -23,7 +23,7 @@ exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. - Copyright 2017-2019 Telegram Systems LLP + Copyright 2017-2020 Telegram Systems LLP */ #include "blockchain-explorer-http.hpp" #include "block/block-db.h" @@ -36,6 +36,8 @@ #include "block/mc-config.h" #include "ton/ton-shard.h" +bool local_scripts{false}; + HttpAnswer& HttpAnswer::operator<<(AddressCell addr_c) { ton::WorkchainId wc; ton::StdSmcAddress addr; @@ -425,7 +427,7 @@ HttpAnswer& HttpAnswer::operator<<(AccountCell acc_c) { HttpAnswer& HttpAnswer::operator<<(BlockHeaderCell head_c) { *this << "
"; - vm::CellSlice cs{vm::NoVm{}, head_c.root}; + vm::CellSlice cs{vm::NoVm(), head_c.root}; auto block_id = head_c.block_id; try { auto virt_root = vm::MerkleProof::virtualize(head_c.root, 1); @@ -676,13 +678,17 @@ std::string HttpAnswer::header() { "maximum-scale=1.0, user-scalable=no\" />\n" << "\n" << "\n" - << "\n" + << "\n" << "" - << "\n" + << "\n" << "\n" - << "\n" + << "\n" << "\n" - << "\n" + << "\n" << "\n" << "
\n" << "