Add Tonlib emulator libs (#744)

* Use global libraries in RunEmulator

* Tonlib method smc.getLibrariesExt

* Process missing transaction in getStateByTransaction

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>

tl/generate/scheme/tonlib_api.tl

@@ -184,6 +184,10 @@ smc.runResult gas_used:int53 stack:vector<tvm.StackEntry> exit_code:int32 = smc.
 smc.libraryEntry hash:int256 data:bytes = smc.LibraryEntry;
 smc.libraryResult result:(vector smc.libraryEntry) = smc.LibraryResult;
 
+smc.libraryQueryExt.one hash:int256 = smc.LibraryQueryExt;
+smc.libraryQueryExt.scanBoc boc:bytes max_libs:int32 = smc.LibraryQueryExt;
+smc.libraryResultExt dict_boc:bytes libs_ok:(vector int256) libs_not_found:(vector int256) = smc.LibraryResultExt;
+
 updateSendLiteServerQuery id:int64 data:bytes = Update;
 updateSyncState sync_state:SyncState = Update;
 
@@ -306,6 +310,7 @@ smc.getState id:int53 = tvm.Cell;
 smc.runGetMethod id:int53 method:smc.MethodId stack:vector<tvm.StackEntry> = smc.RunResult;
 
 smc.getLibraries library_list:(vector int256) = smc.LibraryResult;
+smc.getLibrariesExt list:(vector smc.LibraryQueryExt) = smc.LibraryResultExt;
 
 dns.resolve account_address:accountAddress name:string category:int256 ttl:int32 = dns.Resolved;
 

tonlib/tonlib/TonlibClient.cpp

@@ -61,6 +61,7 @@
 #include "td/utils/port/path.h"
 
 #include "common/util.h"
+#include "td/actor/MultiPromise.h"
 
 template <class Type>
 using lite_api_ptr = ton::lite_api::object_ptr<Type>;
@@ -101,6 +102,11 @@ struct RemoteRunSmcMethodReturnType {
   // libs
 };
 
+struct ScanAndLoadGlobalLibs {
+  td::Ref<vm::Cell> root;
+  using ReturnType = vm::Dictionary;
+};
+
 struct GetPrivateKey {
   KeyStorage::InputKey input_key;
   using ReturnType = KeyStorage::PrivateKey;
@@ -1668,11 +1674,11 @@ class GetShardBlockProof : public td::actor::Actor {
 auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result<lite_api_ptr<ton::lite_api::tonNode_blockIdExt>>;
 auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) -> tonlib_api_ptr<tonlib_api::blocks_shortTxId>;
 
-class RunEmulator : public td::actor::Actor {
+class RunEmulator : public TonlibQueryActor {
  public:
   RunEmulator(ExtClientRef ext_client_ref, int_api::GetAccountStateByTransaction request,
-      td::actor::ActorShared<> parent, td::Promise<td::unique_ptr<AccountState>>&& promise)
+      td::actor::ActorShared<TonlibClient> parent, td::Promise<td::unique_ptr<AccountState>>&& promise)
-    : request_(std::move(request)), parent_(std::move(parent)), promise_(std::move(promise)) {
+    : TonlibQueryActor(std::move(parent)), request_(std::move(request)), promise_(std::move(promise)) {
     client_.set_client(ext_client_ref);
   }
 
@@ -1686,7 +1692,6 @@ class RunEmulator : public td::actor::Actor {
 
   ExtClient client_;
   int_api::GetAccountStateByTransaction request_;
-  td::actor::ActorShared<> parent_;
   td::Promise<td::unique_ptr<AccountState>> promise_;
 
   std::map<td::int64, td::actor::ActorOwn<>> actors_;
@@ -1695,6 +1700,7 @@ class RunEmulator : public td::actor::Actor {
   FullBlockId block_id_;
   td::Ref<vm::Cell> mc_state_root_; // ^ShardStateUnsplit
   td::unique_ptr<AccountState> account_state_;
+  vm::Dictionary global_libraries_{256};
   std::vector<td::Ref<vm::Cell>> transactions_; // std::vector<^Transaction>
 
   size_t count_{0};
@@ -1805,6 +1811,8 @@ class RunEmulator : public td::actor::Actor {
 
       if (bTxes->incomplete_) {
         self->check(self->get_transactions(last_lt));
+      } else {
+        self->check(td::Status::Error("Transaction not found"));
       }
     });
     return td::Status::OK();
@@ -1854,6 +1862,16 @@ class RunEmulator : public td::actor::Actor {
       check(account_state.move_as_error());
     } else {
       account_state_ = account_state.move_as_ok();
+      send_query(int_api::ScanAndLoadGlobalLibs{account_state_->get_raw_state()},
+                 [self = this](td::Result<vm::Dictionary> R) { self->set_global_libraries(std::move(R)); });
+    }
+  }
+
+  void set_global_libraries(td::Result<vm::Dictionary> R) {
+    if (R.is_error()) {
+      check(R.move_as_error());
+    } else {
+      global_libraries_ = R.move_as_ok();
       inc();
     }
   }
@@ -1887,13 +1905,6 @@ class RunEmulator : public td::actor::Actor {
       }
       std::unique_ptr<block::ConfigInfo> config = r_config.move_as_ok();
 
-      block::gen::ShardStateUnsplit::Record shard_state;
-      if (!tlb::unpack_cell(mc_state_root_, shard_state)) {
-        check(td::Status::Error("Failed to unpack masterchain state"));
-        return;
-      }
-      vm::Dictionary libraries(shard_state.r1.libraries->prefetch_ref(), 256);
-
       auto r_shard_account = account_state_->to_shardAccountCellSlice();
       if (r_shard_account.is_error()) {
         check(r_shard_account.move_as_error());
@@ -1915,6 +1926,7 @@ class RunEmulator : public td::actor::Actor {
         check(prev_blocks_info.move_as_error());
         return;
       }
+      vm::Dictionary libraries = global_libraries_;
       emulator::TransactionEmulator trans_emulator(std::move(*config));
       trans_emulator.set_prev_blocks_info(prev_blocks_info.move_as_ok());
       trans_emulator.set_libs(std::move(libraries));
@@ -4376,8 +4388,8 @@ td::Result<vm::StackEntry> from_tonlib_api(tonlib_api::tvm_StackEntry& entry) {
 }
 
 void deep_library_search(std::set<td::Bits256>& set, std::set<vm::Cell::Hash>& visited,
-                         vm::Dictionary& libs, td::Ref<vm::Cell> cell, int depth) {
+                         vm::Dictionary& libs, td::Ref<vm::Cell> cell, int depth, size_t max_libs = 16) {
-  if (depth <= 0 || set.size() >= 16 || visited.size() >= 256) {
+  if (depth <= 0 || set.size() >= max_libs || visited.size() >= 256) {
     return;
   }
   auto ins = visited.insert(cell->get_hash());
@@ -4403,7 +4415,7 @@ void deep_library_search(std::set<td::Bits256>& set, std::set<vm::Cell::Hash>& v
     return;
   }
   for (unsigned int i=0; i<loaded_cell.data_cell->get_refs_cnt(); i++) {
-    deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1);
+    deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1, max_libs);
   }
 }
 
@@ -4459,6 +4471,72 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request,
   return td::Status::OK();
 }
 
+td::Status TonlibClient::do_request(const tonlib_api::smc_getLibrariesExt& request,
+                                    td::Promise<object_ptr<tonlib_api::smc_libraryResultExt>>&& promise) {
+  std::set<td::Bits256> request_libs;
+  for (auto& x : request.list_) {
+    td::Status status = td::Status::OK();
+    downcast_call(*x, td::overloaded([&](tonlib_api::smc_libraryQueryExt_one& one) { request_libs.insert(one.hash_); },
+                                     [&](tonlib_api::smc_libraryQueryExt_scanBoc& scan) {
+                                       std::set<vm::Cell::Hash> visited;
+                                       vm::Dictionary empty{256};
+                                       td::Result<td::Ref<vm::Cell>> r_cell = vm::std_boc_deserialize(scan.boc_);
+                                       if (r_cell.is_error()) {
+                                         status = r_cell.move_as_error();
+                                         return;
+                                       }
+                                       size_t max_libs = scan.max_libs_ < 0 ? (1 << 30) : (size_t)scan.max_libs_;
+                                       std::set<td::Bits256> new_libs;
+                                       deep_library_search(new_libs, visited, empty, r_cell.move_as_ok(), 1024,
+                                                           max_libs);
+                                       request_libs.insert(new_libs.begin(), new_libs.end());
+                                     }));
+    TRY_STATUS(std::move(status));
+  }
+  std::vector<td::Bits256> not_cached;
+  for (const td::Bits256& h : request_libs) {
+    if (libraries.lookup(h).is_null()) {
+      not_cached.push_back(h);
+    }
+  }
+  td::MultiPromise mp;
+  auto ig = mp.init_guard();
+  LOG(DEBUG) << "Requesting " << not_cached.size() << " libraries";
+  for (size_t i = 0; i < not_cached.size(); i += 16) {
+    size_t r = std::min(i + 16, not_cached.size());
+    client_.send_query(
+        ton::lite_api::liteServer_getLibraries(
+            std::vector<td::Bits256>(not_cached.begin() + i, not_cached.begin() + r)),
+        [self = this, promise = ig.get_promise()](
+            td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) mutable {
+          self->process_new_libraries(std::move(r_libraries));
+          promise.set_result(td::Unit());
+        });
+  }
+
+  ig.add_promise(promise.wrap([self = this, libs = std::move(request_libs)](td::Unit&&) {
+    vm::Dictionary dict{256};
+    std::vector<td::Bits256> libs_ok, libs_not_found;
+    for (const auto& h : libs) {
+      auto lib = self->libraries.lookup_ref(h);
+      if (lib.is_null()) {
+        libs_not_found.push_back(h);
+      } else {
+        libs_ok.push_back(h);
+        dict.set_ref(h, lib);
+      }
+    }
+    td::BufferSlice dict_boc;
+    if (!dict.is_empty()) {
+      dict_boc = vm::std_boc_serialize(dict.get_root_cell()).move_as_ok();
+    }
+    return ton::create_tl_object<tonlib_api::smc_libraryResultExt>(dict_boc.as_slice().str(), std::move(libs_ok),
+                                                                   std::move(libs_not_found));
+  }));
+
+  return td::Status::OK();
+}
+
 td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request,
                                     td::Promise<object_ptr<tonlib_api::smc_runResult>>&& promise) {
   auto it = smcs_.find(request.id_);
@@ -4496,37 +4574,14 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request,
       std::vector<td::Bits256> libraryList{librarySet.begin(), librarySet.end()};
       if (libraryList.size() > 0) {
         LOG(DEBUG) << "Requesting found libraries in code (" << libraryList.size() << ")";
-        self->client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(libraryList)),
+        self->client_.send_query(
-                    [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)]
+            ton::lite_api::liteServer_getLibraries(std::move(libraryList)),
-                    (td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) mutable
+            [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)](
-        {
+                td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) mutable {
-          if (r_libraries.is_error()) {
+              self->process_new_libraries(std::move(r_libraries));
-            LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string();
+              self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise));
-          } else {
+            });
-            auto libraries = r_libraries.move_as_ok();
+      } else {
-            bool updated = false;
-            for (auto& lr : libraries->result_) {
-              auto contents = vm::std_boc_deserialize(lr->data_);
-              if (contents.is_ok() && contents.ok().not_null()) {
-                if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) {
-                  LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex();
-                  continue;
-                }
-                self->libraries.set_ref(lr->hash_, contents.move_as_ok());
-                updated = true;
-                LOG(DEBUG) << "registered library " << lr->hash_.to_hex();
-              } else {
-                LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex();
-              }
-            }
-            if (updated) {
-              self->store_libs_to_disk();
-            }
-          }
-          self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise));
-        });
-      }
-      else {
         self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise));
       }
     }
@@ -4537,6 +4592,33 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request,
   return td::Status::OK();
 }
 
+void TonlibClient::process_new_libraries(
+    td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) {
+  if (r_libraries.is_error()) {
+    LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string();
+  } else {
+    auto new_libraries = r_libraries.move_as_ok();
+    bool updated = false;
+    for (auto& lr : new_libraries->result_) {
+      auto contents = vm::std_boc_deserialize(lr->data_);
+      if (contents.is_ok() && contents.ok().not_null()) {
+        if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) {
+          LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex();
+          continue;
+        }
+        libraries.set_ref(lr->hash_, contents.move_as_ok());
+        updated = true;
+        LOG(DEBUG) << "registered library " << lr->hash_.to_hex();
+      } else {
+        LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex();
+      }
+    }
+    if (updated) {
+      store_libs_to_disk();
+    }
+  }
+}
+
 void TonlibClient::perform_smc_execution(td::Ref<ton::SmartContract> smc, ton::SmartContract::Args args,
                                          td::Promise<object_ptr<tonlib_api::smc_runResult>>&& promise) {
 
@@ -5622,6 +5704,30 @@ void TonlibClient::store_libs_to_disk() {  // NB: Dictionary.get_root_cell does
   LOG(DEBUG) << "stored libraries to disk cache";
 }
 
+td::Status TonlibClient::do_request(const int_api::ScanAndLoadGlobalLibs& request, td::Promise<vm::Dictionary> promise) {
+  if (request.root.is_null()) {
+    promise.set_value(vm::Dictionary{256});
+    return td::Status::OK();
+  }
+  std::set<td::Bits256> to_load;
+  std::set<vm::Cell::Hash> visited;
+  deep_library_search(to_load, visited, libraries, request.root, 24);
+  if (to_load.empty()) {
+    promise.set_result(libraries);
+    return td::Status::OK();
+  }
+  std::vector<td::Bits256> to_load_list(to_load.begin(), to_load.end());
+  LOG(DEBUG) << "Requesting found libraries in account state (" << to_load_list.size() << ")";
+  client_.send_query(
+      ton::lite_api::liteServer_getLibraries(std::move(to_load_list)),
+      [self = this, promise = std::move(promise)](
+          td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) mutable {
+        self->process_new_libraries(std::move(r_libraries));
+        promise.set_result(self->libraries);
+      });
+  return td::Status::OK();
+}
+
 template <class P>
 td::Status TonlibClient::do_request(const tonlib_api::runTests& request, P&&) {
   UNREACHABLE();

tonlib/tonlib/TonlibClient.h

@@ -45,6 +45,7 @@ struct GetDnsResolver;
 struct SendMessage;
 struct RemoteRunSmcMethod;
 struct RemoteRunSmcMethodReturnType;
+struct ScanAndLoadGlobalLibs;
 
 inline std::string to_string(const int_api::SendMessage&) {
   return "Send message";
@@ -330,6 +331,9 @@ class TonlibClient : public td::actor::Actor {
   td::Status do_request(const tonlib_api::smc_getLibraries& request,
                         td::Promise<object_ptr<tonlib_api::smc_libraryResult>>&& promise);
 
+  td::Status do_request(const tonlib_api::smc_getLibrariesExt& request,
+                        td::Promise<object_ptr<tonlib_api::smc_libraryResultExt>>&& promise);
+
   td::Status do_request(const tonlib_api::dns_resolve& request,
                         td::Promise<object_ptr<tonlib_api::dns_resolved>>&& promise);
 
@@ -341,6 +345,8 @@ class TonlibClient : public td::actor::Actor {
   td::Status do_request(tonlib_api::pchan_unpackPromise& request,
                         td::Promise<object_ptr<tonlib_api::pchan_promise>>&& promise);
 
+  void process_new_libraries(
+      td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries);
   void perform_smc_execution(td::Ref<ton::SmartContract> smc, ton::SmartContract::Args args,
                              td::Promise<object_ptr<tonlib_api::smc_runResult>>&& promise);
 
@@ -408,5 +414,7 @@ class TonlibClient : public td::actor::Actor {
 
   td::Status guess_revisions(std::vector<Target> targets,
                              td::Promise<object_ptr<tonlib_api::accountRevisionList>>&& promise);
+
+  td::Status do_request(const int_api::ScanAndLoadGlobalLibs& request, td::Promise<vm::Dictionary> promise);
 };
 }  // namespace tonlib