TON Storage utilities (#564)

* Rename chunk to piece in MerkleTree for consistency

* Refactor PeerManager

* Make PeerState thread-safe

* Download torrent by hash

* First version of storage daemon

* Download torrents partially

* Improve storing and loading torrent state in DB

* Rewrite MerkleTree

* "Remove torrent" in storage daemon

* Process errors, fix bugs in storage

* Move TonlibClientWrapper from rldp-http-proxy to tonlib

* Initial version of storage provider

* Move interaction with contracts to smc-util

* Improve TonlibClientWrapper interface

* Various improvements in storage provider

* Fix TorrentCreator.cpp

* Improve interface for partial download

* Client mode in storage-daemon

* Improve interface of storage-daemon-cli

* Fix calculating speed, show peers in storage-daemon

* Use permanent adnl id in storage daemon

* Fix sending large "storage.addUpdate" messages

* Improve printing torrents in cli

* Update tlo

* Fix RldpSender::on_ack

* Update storage provider

* Add "address" parameter to get-provider-params

* Allow client to close storage contract

* Limit torrent description

* Add more logs to storage provider

* smc.forget tonlib method

* Use smc.forget in storage daemon

* Optimize sending messages in smc-util.cpp

* Fix verbosity, remove excessive logs

* Json output in storage-daemon-cli

* Update storage provider contracts

* Fix rldp2 acks

* Change verbosity of logs in rldp2

* Update help and output of commands and in storage-daemon-cli

Co-authored-by: SpyCheese <mikle98@yandex.ru>

tonlib/CMakeLists.txt

@@ -17,6 +17,7 @@ set(TONLIB_SOURCE
   tonlib/LastConfig.cpp
   tonlib/Logging.cpp
   tonlib/TonlibClient.cpp
+  tonlib/TonlibClientWrapper.cpp
   tonlib/utils.cpp
 
   tonlib/Client.h
@@ -32,6 +33,7 @@ set(TONLIB_SOURCE
   tonlib/Logging.h
   tonlib/TonlibCallback.h
   tonlib/TonlibClient.h
+  tonlib/TonlibClientWrapper.h
   tonlib/utils.h
 
   tonlib/keys/bip39.cpp

tonlib/tonlib/TonlibClient.cpp

@@ -2533,7 +2533,7 @@ struct ToRawTransactions {
     auto body_hash = body_cell->get_hash().as_slice().str();
 
     td::Ref<vm::Cell> init_state_cell;
-    auto& init_state_cs = message.init.write(); 
+    auto& init_state_cs = message.init.write();
     if (init_state_cs.fetch_ulong(1) == 1) {
       if (init_state_cs.fetch_long(1) == 0) {
         init_state_cell = vm::CellBuilder().append_cellslice(init_state_cs).finalize();
@@ -2542,7 +2542,7 @@ struct ToRawTransactions {
       }
     }
 
-    auto get_data = [body = std::move(body), body_cell = std::move(body_cell), 
+    auto get_data = [body = std::move(body), body_cell = std::move(body_cell),
                      init_state_cell = std::move(init_state_cell), this](td::Slice salt) mutable {
       tonlib_api::object_ptr<tonlib_api::msg_Data> data;
       if (try_decode_messages_ && body->size() >= 32 && static_cast<td::uint32>(body->prefetch_long(32)) <= 1) {
@@ -2731,7 +2731,7 @@ td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash&
                                     td::Promise<object_ptr<tonlib_api::raw_extMessageInfo>>&& promise) {
   TRY_RESULT_PREFIX(body, vm::std_boc_deserialize(request.body_), TonlibError::InvalidBagOfCells("body"));
   auto hash = body->get_hash().as_slice().str();
-  make_request(int_api::SendMessage{std::move(body)}, 
+  make_request(int_api::SendMessage{std::move(body)},
     promise.wrap([hash = std::move(hash)](auto res) {
       return tonlib_api::make_object<tonlib_api::raw_extMessageInfo>(std::move(hash));
     }));
@@ -2859,7 +2859,7 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request,
 
   auto actor_id = actor_id_++;
   actors_[actor_id] = td::actor::create_actor<GetTransactionHistory>(
-      "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), 
+      "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id),
       promise.wrap([private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable {
         return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x));
       }));
@@ -3710,6 +3710,17 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_load& request,
   return td::Status::OK();
 }
 
+td::Status TonlibClient::do_request(const tonlib_api::smc_forget& request,
+                                    td::Promise<object_ptr<tonlib_api::ok>>&& promise) {
+  auto it = smcs_.find(request.id_);
+  if (it == smcs_.end()) {
+    return TonlibError::InvalidSmcId();
+  }
+  smcs_.erase(it);
+  promise.set_value(tonlib_api::make_object<tonlib_api::ok>());
+  return td::Status::OK();
+}
+
 td::Status TonlibClient::do_request(const tonlib_api::smc_getCode& request,
                                     td::Promise<object_ptr<tonlib_api::tvm_cell>>&& promise) {
   auto it = smcs_.find(request.id_);
@@ -3890,7 +3901,7 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request,
     return td::Status::OK();
   }
 
-  client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(not_cached_hashes)), 
+  client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(not_cached_hashes)),
                      promise.wrap([self=this, result_entries = std::move(result_entries)]
                                   (td::Result<ton::lite_api::object_ptr<ton::lite_api::liteServer_libraryResult>> r_libraries) mutable
     {
@@ -4632,7 +4643,7 @@ td::Status TonlibClient::do_request(const tonlib_api::getConfigParam& request,
   std::vector<int32_t> params = { param };
 
   client_.send_query(ton::lite_api::liteServer_getConfigParams(0, std::move(lite_block), std::move(params)),
-                     promise.wrap([block, param](auto r_config) { 
+                     promise.wrap([block, param](auto r_config) {
     auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(),
                                                   r_config->config_proof_.as_slice());
     if (state.is_error()) {
@@ -4716,7 +4727,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& reques
 
 auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid)
     -> tonlib_api_ptr<tonlib_api::blocks_shortTxId> {
-  return tonlib_api::make_object<tonlib_api::blocks_shortTxId>( 
+  return tonlib_api::make_object<tonlib_api::blocks_shortTxId>(
       txid.mode_, txid.account_.as_slice().str(), txid.lt_, txid.hash_.as_slice().str());
 }
 

tonlib/tonlib/TonlibClient.h

@@ -305,6 +305,7 @@ class TonlibClient : public td::actor::Actor {
   td::Result<tonlib_api::object_ptr<tonlib_api::smc_info>> get_smc_info(td::int64 id);
   void finish_load_smc(td::unique_ptr<AccountState> query, td::Promise<object_ptr<tonlib_api::smc_info>>&& promise);
   td::Status do_request(const tonlib_api::smc_load& request, td::Promise<object_ptr<tonlib_api::smc_info>>&& promise);
+  td::Status do_request(const tonlib_api::smc_forget& request, td::Promise<object_ptr<tonlib_api::ok>>&& promise);
   td::Status do_request(const tonlib_api::smc_getCode& request,
                         td::Promise<object_ptr<tonlib_api::tvm_cell>>&& promise);
   td::Status do_request(const tonlib_api::smc_getData& request,

tonlib/tonlib/TonlibClientWrapper.cpp

@@ -0,0 +1,70 @@
+/*
+    This file is part of TON Blockchain source code.
+
+    TON Blockchain is free software; you can redistribute it and/or
+    modify it under the terms of the GNU General Public License
+    as published by the Free Software Foundation; either version 2
+    of the License, or (at your option) any later version.
+
+    TON Blockchain is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with TON Blockchain.  If not, see <http://www.gnu.org/licenses/>.
+
+    In addition, as a special exception, the copyright holders give permission
+    to link the code of portions of this program with the OpenSSL library.
+    You must obey the GNU General Public License in all respects for all
+    of the code used other than OpenSSL. If you modify file(s) with this
+    exception, you may extend this exception to your version of the file(s),
+    but you are not obligated to do so. If you do not wish to do so, delete this
+    exception statement from your version. If you delete this exception statement
+    from all source files in the program, then also delete it here.
+*/
+#include "TonlibClientWrapper.h"
+
+namespace tonlib {
+
+TonlibClientWrapper::TonlibClientWrapper(ton::tl_object_ptr<tonlib_api::options> options)
+    : options_(std::move(options)) {
+}
+
+void TonlibClientWrapper::start_up() {
+  class Cb : public tonlib::TonlibCallback {
+   public:
+    explicit Cb(td::actor::ActorId<TonlibClientWrapper> self_id) : self_id_(self_id) {
+    }
+    void on_result(std::uint64_t id, tonlib_api::object_ptr<tonlib_api::Object> result) override {
+      td::actor::send_closure(self_id_, &TonlibClientWrapper::receive_request_result, id, std::move(result));
+    }
+    void on_error(std::uint64_t id, tonlib_api::object_ptr<tonlib_api::error> error) override {
+      td::actor::send_closure(self_id_, &TonlibClientWrapper::receive_request_result, id,
+                              td::Status::Error(error->code_, std::move(error->message_)));
+    }
+
+   private:
+    td::actor::ActorId<TonlibClientWrapper> self_id_;
+  };
+
+  tonlib_client_ = td::actor::create_actor<tonlib::TonlibClient>("tonlibclient", td::make_unique<Cb>(actor_id(this)));
+  auto init = tonlib_api::make_object<tonlib_api::init>(std::move(options_));
+  auto P = td::PromiseCreator::lambda(
+      [](td::Result<tonlib_api::object_ptr<tonlib_api::options_info>> R) mutable { R.ensure(); });
+  send_request(std::move(init), std::move(P));
+}
+
+void TonlibClientWrapper::receive_request_result(td::uint64 id,
+                                                 td::Result<tonlib_api::object_ptr<tonlib_api::Object>> R) {
+  if (id == 0) {
+    return;
+  }
+  auto it = requests_.find(id);
+  CHECK(it != requests_.end());
+  auto promise = std::move(it->second);
+  requests_.erase(it);
+  promise.set_result(std::move(R));
+}
+
+}  // namespace tonlib

tonlib/tonlib/TonlibClientWrapper.h

@@ -0,0 +1,61 @@
+/*
+    This file is part of TON Blockchain source code.
+
+    TON Blockchain is free software; you can redistribute it and/or
+    modify it under the terms of the GNU General Public License
+    as published by the Free Software Foundation; either version 2
+    of the License, or (at your option) any later version.
+
+    TON Blockchain is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with TON Blockchain.  If not, see <http://www.gnu.org/licenses/>.
+
+    In addition, as a special exception, the copyright holders give permission
+    to link the code of portions of this program with the OpenSSL library.
+    You must obey the GNU General Public License in all respects for all
+    of the code used other than OpenSSL. If you modify file(s) with this
+    exception, you may extend this exception to your version of the file(s),
+    but you are not obligated to do so. If you do not wish to do so, delete this
+    exception statement from your version. If you delete this exception statement
+    from all source files in the program, then also delete it here.
+*/
+#pragma once
+#include "td/actor/actor.h"
+#include "auto/tl/tonlib_api.hpp"
+#include "tonlib/tonlib/TonlibClient.h"
+
+namespace tonlib {
+
+class TonlibClientWrapper : public td::actor::Actor {
+ public:
+  explicit TonlibClientWrapper(ton::tl_object_ptr<tonlib_api::options> options);
+
+  void start_up() override;
+
+  template <typename F>
+  void send_request(tonlib_api::object_ptr<F> obj, td::Promise<typename F::ReturnType> promise) {
+    auto id = next_request_id_++;
+    auto P = promise.wrap([](tonlib_api::object_ptr<tonlib_api::Object> x) -> td::Result<typename F::ReturnType> {
+      if (x->get_id() != F::ReturnType::element_type::ID) {
+        return td::Status::Error("Invalid response from tonlib");
+      }
+      return ton::move_tl_object_as<typename F::ReturnType::element_type>(std::move(x));
+    });
+    CHECK(requests_.emplace(id, std::move(P)).second);
+    td::actor::send_closure(tonlib_client_, &tonlib::TonlibClient::request, id, std::move(obj));
+  }
+
+ private:
+  void receive_request_result(td::uint64 id, td::Result<tonlib_api::object_ptr<tonlib_api::Object>> R);
+
+  ton::tl_object_ptr<tonlib_api::options> options_;
+  td::actor::ActorOwn<tonlib::TonlibClient> tonlib_client_;
+  std::map<td::uint64, td::Promise<tonlib_api::object_ptr<tonlib_api::Object>>> requests_;
+  td::uint64 next_request_id_{1};
+};
+
+}  // namespace tonlib