Support libraries in LS and request+caching in TLC

crypto/block/mc-config.cpp

@@ -2100,7 +2100,7 @@ Ref<vm::Cell> ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const {
     return {};
   }
   auto csr = libraries_dict_->lookup(root_hash, 256);
-  if (csr.is_null() || csr->prefetch_ulong(8) != 0 || !csr->have_refs()) {  // shared_lib_descr$00 lib:^Cell
+  if (csr.is_null() || csr->prefetch_ulong(2) != 0 || !csr->have_refs()) {  // shared_lib_descr$00 lib:^Cell
     return {};
   }
   auto lib = csr->prefetch_ref();

crypto/smc-envelope/SmartContract.cpp

@@ -129,8 +129,9 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref<vm::Stac
   vm::VmState vm{state.code, std::move(stack), gas, 1, state.data, log};
   vm.set_c7(std::move(c7));
   vm.set_chksig_always_succeed(ignore_chksig);
-  if (!libraries.is_null())
+  if (!libraries.is_null()) {
     vm.register_library_collection(libraries);
+  }
   try {
     res.code = ~vm.run();
   } catch (...) {

tl/generate/scheme/lite_api.tl

@@ -25,6 +25,7 @@ adnl.message.answer query_id:int256 answer:bytes = adnl.Message;
 liteServer.error code:int message:string = liteServer.Error; 
 
 liteServer.accountId workchain:int id:int256 = liteServer.AccountId;
+liteServer.libraryEntry hash:int256 data:bytes = liteServer.LibraryEntry;
 
 liteServer.masterchainInfo last:tonNode.blockIdExt state_root_hash:int256 init:tonNode.zeroStateIdExt = liteServer.MasterchainInfo;
 liteServer.masterchainInfoExt mode:# version:int capabilities:long last:tonNode.blockIdExt last_utime:int now:int state_root_hash:int256 init:tonNode.zeroStateIdExt = liteServer.MasterchainInfoExt;
@@ -50,6 +51,7 @@ liteServer.blockLinkForward to_key_block:Bool from:tonNode.blockIdExt to:tonNode
 liteServer.partialBlockProof complete:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt steps:(vector liteServer.BlockLink) = liteServer.PartialBlockProof;
 liteServer.configInfo mode:# id:tonNode.blockIdExt state_proof:bytes config_proof:bytes = liteServer.ConfigInfo;
 liteServer.validatorStats mode:# id:tonNode.blockIdExt count:int complete:Bool state_proof:bytes data_proof:bytes = liteServer.ValidatorStats;
+liteServer.libraryResult result:(vector liteServer.libraryEntry) = liteServer.LibraryResult;
 
 liteServer.debug.verbosity value:int = liteServer.debug.Verbosity;
 
@@ -75,6 +77,7 @@ liteServer.getBlockProof mode:# known_block:tonNode.blockIdExt target_block:mode
 liteServer.getConfigAll mode:# id:tonNode.blockIdExt = liteServer.ConfigInfo;
 liteServer.getConfigParams mode:# id:tonNode.blockIdExt param_list:(vector int) = liteServer.ConfigInfo;
 liteServer.getValidatorStats#091a58bc mode:# id:tonNode.blockIdExt limit:int start_after:mode.0?int256 modified_after:mode.2?int = liteServer.ValidatorStats;
+liteServer.getLibraries library_list:(vector int256) = liteServer.LibraryResult;
 
 liteServer.queryPrefix = Object; 
 liteServer.query data:bytes = Object;

tonlib/tonlib/TonlibClient.cpp

@@ -1889,6 +1889,8 @@ td::Status TonlibClient::do_request(const tonlib_api::init& request,
   TRY_RESULT(kv, std::move(r_kv));
   kv_ = std::shared_ptr<KeyValue>(kv.release());
 
+  load_libs_from_disk();
+
   key_storage_.set_key_value(kv_);
   last_block_storage_.set_key_value(kv_);
   auto res = tonlib_api::make_object<tonlib_api::options_info>();
@@ -3474,6 +3476,38 @@ td::Result<vm::StackEntry> from_tonlib_api(tonlib_api::tvm_StackEntry& entry) {
           }));
 }
 
+void deep_library_search(std::set<td::Bits256>& set, std::set<vm::Cell::Hash>& visited,
+                         vm::Dictionary libs, td::Ref<vm::Cell> cell, int depth) {
+  if (depth <= 0 || set.size() >= 16 || visited.size() >= 256) {
+    return;
+  }
+  auto ins = visited.insert(cell->get_hash());
+  if (!ins.second) {
+    return;  // already visited this cell
+  }
+  auto r_loaded_cell = cell->load_cell();
+  if (r_loaded_cell.is_error()) {
+    return;
+  }
+  auto loaded_cell = r_loaded_cell.move_as_ok();
+  if (loaded_cell.data_cell->is_special()) {
+    if (loaded_cell.data_cell->special_type() == vm::DataCell::SpecialType::Library) {
+      vm::CellSlice cs(std::move(loaded_cell));
+      if (cs.size() != vm::Cell::hash_bits + 8) {
+        return;
+      }
+      auto key = td::Bits256(cs.data_bits() + 8);
+      if (libs.lookup(key).is_null()) {
+        set.insert(key);
+      }
+    }
+    return;
+  }
+  for (unsigned int i=0; i<loaded_cell.data_cell->get_refs_cnt(); i++) {
+    deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1);
+  }
+}
+
 td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request,
                                     td::Promise<object_ptr<tonlib_api::smc_runResult>>&& promise) {
   auto it = smcs_.find(request.id_);
@@ -3497,30 +3531,120 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request,
   args.set_now(it->second->get_sync_time());
   args.set_address(it->second->get_address());
 
-  auto code = smc->get_state().code;
-  if (code.not_null()) {
-    std::vector<td::Bits256> libraryList;
-  }
-
-  args.set_libraries(libraries);
-
-  client_.with_last_config([smc=std::move(smc), args=std::move(args),
-                            promise = std::move(promise)](td::Result<LastConfigState> r_state) mutable {
+  client_.with_last_config([self = this, smc = std::move(smc), args = std::move(args), promise = std::move(promise)
+  ](td::Result<LastConfigState> r_state) mutable {
     TRY_RESULT_PROMISE(promise, state, std::move(r_state));
     args.set_config(state.config);
 
-    auto res = smc->run_get_method(std::move(args));
-
-    // smc.runResult gas_used:int53 stack:vector<tvm.StackEntry> exit_code:int32 = smc.RunResult;
-    std::vector<object_ptr<tonlib_api::tvm_StackEntry>> res_stack;
-    for (auto& entry : res.stack->as_span()) {
-      res_stack.push_back(to_tonlib_api(entry));
+    auto code = smc->get_state().code;
+    if (code.not_null()) {
+      std::set<td::Bits256> librarySet;
+      std::set<vm::Cell::Hash> visited;
+      deep_library_search(librarySet, visited, self->libraries, code, 24);
+      std::vector<td::Bits256> libraryList{librarySet.begin(), librarySet.end()};
+      if (libraryList.size() > 0) {
+        LOG(DEBUG) << "Requesting found libraries in code (" << libraryList.size() << ")";
+        self->client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(libraryList)),
+                    [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)]
+                    (td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) mutable
+        {
+          if (r_libraries.is_error()) {
+            LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string();
+          } else {
+            auto libraries = r_libraries.move_as_ok();
+            bool updated = false;
+            for (auto& lr : libraries->result_) {
+              auto contents = vm::std_boc_deserialize(lr->data_);
+              if (contents.is_ok() && contents.ok().not_null()) {
+                if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) {
+                  LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex();
+                  continue;
+                }
+                self->libraries.set_ref(lr->hash_, contents.move_as_ok());
+                updated = true;
+                LOG(DEBUG) << "registered library " << lr->hash_.to_hex();
+              } else {
+                LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex();
+              }
+            }
+            if (updated) {
+              self->store_libs_to_disk();
+            }
+          }
+          self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise));
+        });
+      }
+      else {
+        self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise));
+      }
+    }
+    else {
+      self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise));
     }
-    promise.set_value(tonlib_api::make_object<tonlib_api::smc_runResult>(res.gas_used, std::move(res_stack), res.code));
   });
   return td::Status::OK();
 }
 
+void TonlibClient::perform_smc_execution(td::Ref<ton::SmartContract> smc, ton::SmartContract::Args args,
+                                         td::Promise<object_ptr<tonlib_api::smc_runResult>>&& promise) {
+
+  args.set_libraries(libraries);
+
+  auto res = smc->run_get_method(args);
+
+  // smc.runResult gas_used:int53 stack:vector<tvm.StackEntry> exit_code:int32 = smc.RunResult;
+  std::vector<object_ptr<tonlib_api::tvm_StackEntry>> res_stack;
+  for (auto& entry : res.stack->as_span()) {
+    res_stack.push_back(to_tonlib_api(entry));
+  }
+
+  if (res.missing_library.not_null()) {
+    td::Bits256 hash = res.missing_library;
+    LOG(DEBUG) << "Requesting missing library: " << hash.to_hex();
+    std::vector<td::Bits256> req = {std::move(hash)};
+    client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(req)),
+                [self = this, res = std::move(res), res_stack = std::move(res_stack), hash = std::move(hash),
+                 smc = std::move(smc), args = std::move(args), promise = std::move(promise)]
+                (td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) mutable
+    {
+      if (r_libraries.is_error()) {
+        LOG(WARNING) << "cannot obtain missing library: " << r_libraries.move_as_error().to_string();
+        promise.set_value(tonlib_api::make_object<tonlib_api::smc_runResult>(res.gas_used, std::move(res_stack), res.code));
+        return;
+      }
+      bool found = false, updated = false;
+      auto libraries = r_libraries.move_as_ok();
+      for (auto& lr : libraries->result_) {
+        auto contents = vm::std_boc_deserialize(lr->data_);
+        if (contents.is_ok() && contents.ok().not_null()) {
+          if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) {
+            LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex();
+            continue;
+          }
+          found |= (lr->hash_ == hash);
+          updated = true;
+          self->libraries.set_ref(lr->hash_, contents.move_as_ok());
+          LOG(DEBUG) << "registered library " << lr->hash_.to_hex();
+        } else {
+          LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex();
+        }
+      }
+      if (updated) {
+        self->store_libs_to_disk();
+      }
+      if (!found) {
+        LOG(WARNING) << "cannot obtain library " << hash.to_hex() << ", it may not exist";
+        promise.set_value(tonlib_api::make_object<tonlib_api::smc_runResult>(res.gas_used, std::move(res_stack), res.code));
+      } else {
+        self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise));
+      }
+    });
+  }
+  else {
+    promise.set_value(tonlib_api::make_object<tonlib_api::smc_runResult>(res.gas_used, std::move(res_stack), res.code));
+  }
+}
+
 td::Result<tonlib_api::object_ptr<tonlib_api::dns_EntryData>> to_tonlib_api(
     const ton::ManualDns::EntryData& entry_data) {
   td::Result<tonlib_api::object_ptr<tonlib_api::dns_EntryData>> res;
@@ -4246,6 +4370,28 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req
   return td::Status::OK();
 }
 
+void TonlibClient::load_libs_from_disk() {
+  LOG(DEBUG) << "loading libraries from disk cache";
+  auto r_data = kv_->get("tonlib.libcache");
+  if (r_data.is_error()) {
+    return;
+  }
+  auto r_dict = vm::std_boc_deserialize(r_data.move_as_ok(), true);
+  if (r_dict.is_error()) {
+    return;
+  }
+  libraries = vm::Dictionary(vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice(
+                                                                   r_dict.move_as_ok())).finalize()), 256);
+  // int n = 0; for (auto&& lr : libraries) n++;
+  LOG(DEBUG) << "loaded libraries from disk cache";
+}
+
+void TonlibClient::store_libs_to_disk() {  // NB: Dictionary.get_root_cell does not compute_root, and it is protected
+  kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root())
+                                                        .finalize()).move_as_ok().as_slice());
+  // int n = 0; for (auto&& lr : libraries) n++;
+  LOG(DEBUG) << "stored libraries to disk cache";
+}
 
 template <class P>
 td::Status TonlibClient::do_request(const tonlib_api::runTests& request, P&&) {

tonlib/tonlib/TonlibClient.h

@@ -322,7 +322,8 @@ class TonlibClient : public td::actor::Actor {
   td::Status do_request(tonlib_api::pchan_unpackPromise& request,
                         td::Promise<object_ptr<tonlib_api::pchan_promise>>&& promise);
 
-
+  void perform_smc_execution(td::Ref<ton::SmartContract> smc, ton::SmartContract::Args args,
+                             td::Promise<object_ptr<tonlib_api::smc_runResult>>&& promise);
 
   void do_dns_request(std::string name, td::int32 category, td::int32 ttl, td::optional<ton::BlockIdExt> block_id,
                       block::StdAddress address, td::Promise<object_ptr<tonlib_api::dns_resolved>>&& promise);
@@ -358,6 +359,9 @@ class TonlibClient : public td::actor::Actor {
 
   void proxy_request(td::int64 query_id, std::string data);
 
+  void load_libs_from_disk();
+  void store_libs_to_disk();
+
   friend class TonlibQueryActor;
   struct Target {
     bool can_be_empty{true};

validator/impl/liteserver.cpp

@@ -198,6 +198,9 @@ void LiteQuery::start_up() {
             this->perform_runSmcMethod(ton::create_block_id(q.id_), static_cast<WorkchainId>(q.account_->workchain_),
                                        q.account_->id_, q.mode_, q.method_id_, std::move(q.params_));
           },
+          [&](lite_api::liteServer_getLibraries& q) {
+            this->perform_getLibraries(q.library_list_);
+          },
           [&](auto& obj) { this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); }));
 }
 
@@ -781,6 +784,73 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St
   perform_getAccountState(blkid, workchain, addr, mode | 0x10000);
 }
 
+void LiteQuery::perform_getLibraries(std::vector<td::Bits256> library_list) {
+  LOG(INFO) << "started a getLibraries(<list of " << library_list.size() << " parameters>) liteserver query";
+  if (library_list.size() > 16) {
+    LOG(INFO) << "too many libraries requested, returning only first 16";
+    library_list.resize(16);
+  }
+  sort( library_list.begin(), library_list.end() );
+  library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() );
+  td::actor::send_closure_later(
+      manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block,
+      [Self = actor_id(this), library_list](td::Result<std::pair<Ref<ton::validator::MasterchainState>, BlockIdExt>> res) -> void {
+        if (res.is_error()) {
+          td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error());
+        } else {
+          auto pair = res.move_as_ok();
+          td::actor::send_closure_later(Self, &LiteQuery::continue_getLibraries, std::move(pair.first),
+                                        pair.second, library_list);
+        }
+      });
+}
+
+void LiteQuery::continue_getLibraries(Ref<ton::validator::MasterchainState> mc_state, BlockIdExt blkid, std::vector<td::Bits256> library_list) {
+  LOG(INFO) << "obtained last masterchain block = " << blkid.to_str();
+  base_blk_id_ = blkid;
+  CHECK(mc_state.not_null());
+  mc_state_ = Ref<MasterchainStateQ>(std::move(mc_state));
+  CHECK(mc_state_.not_null());
+
+  auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries);
+  if (rconfig.is_error()) {
+    fatal_error("cannot extract library list block configuration from masterchain state");
+    return;
+  }
+  auto config = rconfig.move_as_ok();
+
+  if (false) {
+    std::ostringstream os;
+    vm::load_cell_slice(config->get_libraries_root()).print_rec(os);
+    LOG(INFO) << "\n" << os.str();
+
+    auto lib_dict = std::make_unique<vm::Dictionary>(config->get_libraries_root(), 256);
+    for (auto k: *lib_dict) {
+      std::ostringstream oss;
+      k.second->print_rec(oss);
+      LOG(INFO) << "library " << k.first.to_hex(256) << ": \n" << oss.str();
+    }
+  }
+
+  std::vector<ton::tl_object_ptr<ton::lite_api::liteServer_libraryEntry>> a;
+  for (const auto& hash : library_list) {
+    LOG(INFO) << "looking for library " << hash.to_hex();
+    auto libres = config->lookup_library(hash);
+    if (libres.is_null()) {
+      LOG(INFO) << "library lookup result is null";
+      continue;
+    }
+    auto data = vm::std_boc_serialize(libres);
+    if (data.is_error()) {
+      LOG(WARNING) << "library serialization failed: " << data.move_as_error().to_string();
+      continue;
+    }
+    a.push_back(ton::create_tl_object<ton::lite_api::liteServer_libraryEntry>(hash, data.move_as_ok()));
+  }
+  auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_libraryResult>(std::move(a));
+  finish_query(std::move(b));
+}
+
 void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt) {
   LOG(INFO) << "started a getOneTransaction(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ","
             << lt << ") liteserver query";

validator/impl/liteserver.hpp

@@ -115,6 +115,8 @@ class LiteQuery : public td::actor::Actor {
                             td::BufferSlice params);
   void finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice state_proof, Ref<vm::Cell> acc_root,
                            UnixTime gen_utime, LogicalTime gen_lt);
+  void perform_getLibraries(std::vector<td::Bits256> library_list);
+  void continue_getLibraries(Ref<MasterchainState> mc_state, BlockIdExt blkid, std::vector<td::Bits256> library_list);
   void perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt);
   void continue_getOneTransaction();
   void perform_getTransactions(WorkchainId workchain, StdSmcAddress addr, LogicalTime lt, Bits256 hash, unsigned count);