diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index bcfc625d..af08cbd7 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -184,6 +184,10 @@ smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc. smc.libraryEntry hash:int256 data:bytes = smc.LibraryEntry; smc.libraryResult result:(vector smc.libraryEntry) = smc.LibraryResult; +smc.libraryQueryExt.one hash:int256 = smc.LibraryQueryExt; +smc.libraryQueryExt.scanBoc boc:bytes max_libs:int32 = smc.LibraryQueryExt; +smc.libraryResultExt dict_boc:bytes libs_ok:(vector int256) libs_not_found:(vector int256) = smc.LibraryResultExt; + updateSendLiteServerQuery id:int64 data:bytes = Update; updateSyncState sync_state:SyncState = Update; @@ -306,6 +310,7 @@ smc.getState id:int53 = tvm.Cell; smc.runGetMethod id:int53 method:smc.MethodId stack:vector = smc.RunResult; smc.getLibraries library_list:(vector int256) = smc.LibraryResult; +smc.getLibrariesExt list:(vector smc.LibraryQueryExt) = smc.LibraryResultExt; dns.resolve account_address:accountAddress name:string category:int256 ttl:int32 = dns.Resolved; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 023a4953..6644d118 100644 Binary files a/tl/generate/scheme/tonlib_api.tlo and b/tl/generate/scheme/tonlib_api.tlo differ diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index c571cf65..74b427db 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -61,6 +61,7 @@ #include "td/utils/port/path.h" #include "common/util.h" +#include "td/actor/MultiPromise.h" template using lite_api_ptr = ton::lite_api::object_ptr; @@ -101,6 +102,11 @@ struct RemoteRunSmcMethodReturnType { // libs }; +struct ScanAndLoadGlobalLibs { + td::Ref root; + using ReturnType = vm::Dictionary; +}; + struct GetPrivateKey { KeyStorage::InputKey input_key; using ReturnType = KeyStorage::PrivateKey; @@ -1668,11 +1674,11 @@ class GetShardBlockProof : public td::actor::Actor { auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result>; auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) -> tonlib_api_ptr; -class RunEmulator : public td::actor::Actor { +class RunEmulator : public TonlibQueryActor { public: RunEmulator(ExtClientRef ext_client_ref, int_api::GetAccountStateByTransaction request, - td::actor::ActorShared<> parent, td::Promise>&& promise) - : request_(std::move(request)), parent_(std::move(parent)), promise_(std::move(promise)) { + td::actor::ActorShared parent, td::Promise>&& promise) + : TonlibQueryActor(std::move(parent)), request_(std::move(request)), promise_(std::move(promise)) { client_.set_client(ext_client_ref); } @@ -1686,7 +1692,6 @@ class RunEmulator : public td::actor::Actor { ExtClient client_; int_api::GetAccountStateByTransaction request_; - td::actor::ActorShared<> parent_; td::Promise> promise_; std::map> actors_; @@ -1695,6 +1700,7 @@ class RunEmulator : public td::actor::Actor { FullBlockId block_id_; td::Ref mc_state_root_; // ^ShardStateUnsplit td::unique_ptr account_state_; + vm::Dictionary global_libraries_{256}; std::vector> transactions_; // std::vector<^Transaction> size_t count_{0}; @@ -1805,6 +1811,8 @@ class RunEmulator : public td::actor::Actor { if (bTxes->incomplete_) { self->check(self->get_transactions(last_lt)); + } else { + self->check(td::Status::Error("Transaction not found")); } }); return td::Status::OK(); @@ -1854,6 +1862,16 @@ class RunEmulator : public td::actor::Actor { check(account_state.move_as_error()); } else { account_state_ = account_state.move_as_ok(); + send_query(int_api::ScanAndLoadGlobalLibs{account_state_->get_raw_state()}, + [self = this](td::Result R) { self->set_global_libraries(std::move(R)); }); + } + } + + void set_global_libraries(td::Result R) { + if (R.is_error()) { + check(R.move_as_error()); + } else { + global_libraries_ = R.move_as_ok(); inc(); } } @@ -1887,13 +1905,6 @@ class RunEmulator : public td::actor::Actor { } std::unique_ptr config = r_config.move_as_ok(); - block::gen::ShardStateUnsplit::Record shard_state; - if (!tlb::unpack_cell(mc_state_root_, shard_state)) { - check(td::Status::Error("Failed to unpack masterchain state")); - return; - } - vm::Dictionary libraries(shard_state.r1.libraries->prefetch_ref(), 256); - auto r_shard_account = account_state_->to_shardAccountCellSlice(); if (r_shard_account.is_error()) { check(r_shard_account.move_as_error()); @@ -1915,6 +1926,7 @@ class RunEmulator : public td::actor::Actor { check(prev_blocks_info.move_as_error()); return; } + vm::Dictionary libraries = global_libraries_; emulator::TransactionEmulator trans_emulator(std::move(*config)); trans_emulator.set_prev_blocks_info(prev_blocks_info.move_as_ok()); trans_emulator.set_libs(std::move(libraries)); @@ -4376,8 +4388,8 @@ td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { } void deep_library_search(std::set& set, std::set& visited, - vm::Dictionary& libs, td::Ref cell, int depth) { - if (depth <= 0 || set.size() >= 16 || visited.size() >= 256) { + vm::Dictionary& libs, td::Ref cell, int depth, size_t max_libs = 16) { + if (depth <= 0 || set.size() >= max_libs || visited.size() >= 256) { return; } auto ins = visited.insert(cell->get_hash()); @@ -4403,7 +4415,7 @@ void deep_library_search(std::set& set, std::set& v return; } for (unsigned int i=0; iget_refs_cnt(); i++) { - deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1); + deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1, max_libs); } } @@ -4459,6 +4471,72 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request, return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::smc_getLibrariesExt& request, + td::Promise>&& promise) { + std::set request_libs; + for (auto& x : request.list_) { + td::Status status = td::Status::OK(); + downcast_call(*x, td::overloaded([&](tonlib_api::smc_libraryQueryExt_one& one) { request_libs.insert(one.hash_); }, + [&](tonlib_api::smc_libraryQueryExt_scanBoc& scan) { + std::set visited; + vm::Dictionary empty{256}; + td::Result> r_cell = vm::std_boc_deserialize(scan.boc_); + if (r_cell.is_error()) { + status = r_cell.move_as_error(); + return; + } + size_t max_libs = scan.max_libs_ < 0 ? (1 << 30) : (size_t)scan.max_libs_; + std::set new_libs; + deep_library_search(new_libs, visited, empty, r_cell.move_as_ok(), 1024, + max_libs); + request_libs.insert(new_libs.begin(), new_libs.end()); + })); + TRY_STATUS(std::move(status)); + } + std::vector not_cached; + for (const td::Bits256& h : request_libs) { + if (libraries.lookup(h).is_null()) { + not_cached.push_back(h); + } + } + td::MultiPromise mp; + auto ig = mp.init_guard(); + LOG(DEBUG) << "Requesting " << not_cached.size() << " libraries"; + for (size_t i = 0; i < not_cached.size(); i += 16) { + size_t r = std::min(i + 16, not_cached.size()); + client_.send_query( + ton::lite_api::liteServer_getLibraries( + std::vector(not_cached.begin() + i, not_cached.begin() + r)), + [self = this, promise = ig.get_promise()]( + td::Result> r_libraries) mutable { + self->process_new_libraries(std::move(r_libraries)); + promise.set_result(td::Unit()); + }); + } + + ig.add_promise(promise.wrap([self = this, libs = std::move(request_libs)](td::Unit&&) { + vm::Dictionary dict{256}; + std::vector libs_ok, libs_not_found; + for (const auto& h : libs) { + auto lib = self->libraries.lookup_ref(h); + if (lib.is_null()) { + libs_not_found.push_back(h); + } else { + libs_ok.push_back(h); + dict.set_ref(h, lib); + } + } + td::BufferSlice dict_boc; + if (!dict.is_empty()) { + dict_boc = vm::std_boc_serialize(dict.get_root_cell()).move_as_ok(); + } + return ton::create_tl_object(dict_boc.as_slice().str(), std::move(libs_ok), + std::move(libs_not_found)); + })); + + return td::Status::OK(); +} + td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, td::Promise>&& promise) { auto it = smcs_.find(request.id_); @@ -4496,37 +4574,14 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, std::vector libraryList{librarySet.begin(), librarySet.end()}; if (libraryList.size() > 0) { LOG(DEBUG) << "Requesting found libraries in code (" << libraryList.size() << ")"; - self->client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(libraryList)), - [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)] - (td::Result> r_libraries) mutable - { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); - } else { - auto libraries = r_libraries.move_as_ok(); - bool updated = false; - for (auto& lr : libraries->result_) { - auto contents = vm::std_boc_deserialize(lr->data_); - if (contents.is_ok() && contents.ok().not_null()) { - if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { - LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); - continue; - } - self->libraries.set_ref(lr->hash_, contents.move_as_ok()); - updated = true; - LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); - } else { - LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); - } - } - if (updated) { - self->store_libs_to_disk(); - } - } - self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); - }); - } - else { + self->client_.send_query( + ton::lite_api::liteServer_getLibraries(std::move(libraryList)), + [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)]( + td::Result> r_libraries) mutable { + self->process_new_libraries(std::move(r_libraries)); + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + }); + } else { self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); } } @@ -4537,6 +4592,33 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, return td::Status::OK(); } +void TonlibClient::process_new_libraries( + td::Result> r_libraries) { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); + } else { + auto new_libraries = r_libraries.move_as_ok(); + bool updated = false; + for (auto& lr : new_libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + libraries.set_ref(lr->hash_, contents.move_as_ok()); + updated = true; + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + store_libs_to_disk(); + } + } +} + void TonlibClient::perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, td::Promise>&& promise) { @@ -5622,6 +5704,30 @@ void TonlibClient::store_libs_to_disk() { // NB: Dictionary.get_root_cell does LOG(DEBUG) << "stored libraries to disk cache"; } +td::Status TonlibClient::do_request(const int_api::ScanAndLoadGlobalLibs& request, td::Promise promise) { + if (request.root.is_null()) { + promise.set_value(vm::Dictionary{256}); + return td::Status::OK(); + } + std::set to_load; + std::set visited; + deep_library_search(to_load, visited, libraries, request.root, 24); + if (to_load.empty()) { + promise.set_result(libraries); + return td::Status::OK(); + } + std::vector to_load_list(to_load.begin(), to_load.end()); + LOG(DEBUG) << "Requesting found libraries in account state (" << to_load_list.size() << ")"; + client_.send_query( + ton::lite_api::liteServer_getLibraries(std::move(to_load_list)), + [self = this, promise = std::move(promise)]( + td::Result> r_libraries) mutable { + self->process_new_libraries(std::move(r_libraries)); + promise.set_result(self->libraries); + }); + return td::Status::OK(); +} + template td::Status TonlibClient::do_request(const tonlib_api::runTests& request, P&&) { UNREACHABLE(); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index ed81760f..f4651b9d 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -45,6 +45,7 @@ struct GetDnsResolver; struct SendMessage; struct RemoteRunSmcMethod; struct RemoteRunSmcMethodReturnType; +struct ScanAndLoadGlobalLibs; inline std::string to_string(const int_api::SendMessage&) { return "Send message"; @@ -330,6 +331,9 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::smc_getLibraries& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::smc_getLibrariesExt& request, + td::Promise>&& promise); + td::Status do_request(const tonlib_api::dns_resolve& request, td::Promise>&& promise); @@ -341,6 +345,8 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(tonlib_api::pchan_unpackPromise& request, td::Promise>&& promise); + void process_new_libraries( + td::Result> r_libraries); void perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, td::Promise>&& promise); @@ -408,5 +414,7 @@ class TonlibClient : public td::actor::Actor { td::Status guess_revisions(std::vector targets, td::Promise>&& promise); + + td::Status do_request(const int_api::ScanAndLoadGlobalLibs& request, td::Promise promise); }; } // namespace tonlib