diff --git a/CMakeLists.txt b/CMakeLists.txt index eb51cf8f..e07ca089 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,7 +71,7 @@ endif() #BEGIN internal option(TON_ONLY_TONLIB "Use \"ON\" to build only tonlib." OFF) -if (TON_ONLY_TONBLIB) +if (TON_ONLY_TONLIB) set(NOT_TON_ONLY_TONLIB false) else() set(NOT_TON_ONLY_TONLIB true) diff --git a/adnl/adnl-ext-client.cpp b/adnl/adnl-ext-client.cpp index efc71e6f..d7a39d41 100644 --- a/adnl/adnl-ext-client.cpp +++ b/adnl/adnl-ext-client.cpp @@ -176,6 +176,113 @@ td::Status AdnlOutboundConnection::process_packet(td::BufferSlice data) { return td::Status::OK(); } +void AdnlExtMultiClientImpl::start_up() { + for (auto &id : ids_) { + add_server(id.first, id.second, [](td::Result R) {}); + } + ids_.clear(); +} + +void AdnlExtMultiClientImpl::add_server(AdnlNodeIdFull dst, td::IPAddress dst_addr, td::Promise promise) { + for (auto &c : clients_) { + if (c.second->addr == dst_addr) { + promise.set_error(td::Status::Error(ErrorCode::error, "duplicate ip")); + return; + } + } + + auto g = ++generation_; + auto cli = std::make_unique(AdnlExtClient::create(dst, dst_addr, make_callback(g)), dst, dst_addr, g); + clients_[g] = std::move(cli); +} + +void AdnlExtMultiClientImpl::del_server(td::IPAddress dst_addr, td::Promise promise) { + for (auto &c : clients_) { + if (c.second->addr == dst_addr) { + if (c.second->ready) { + total_ready_--; + if (!total_ready_) { + callback_->on_stop_ready(); + } + } + clients_.erase(c.first); + promise.set_value(td::Unit()); + return; + } + } + promise.set_error(td::Status::Error(ErrorCode::error, "ip not found")); +} + +void AdnlExtMultiClientImpl::send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + td::Promise promise) { + if (total_ready_ == 0) { + promise.set_error(td::Status::Error(ErrorCode::notready, "conn not ready")); + return; + } + + std::vector vec; + for (auto &c : clients_) { + if (c.second->ready) { + vec.push_back(c.first); + } + } + CHECK(vec.size() == total_ready_); + + auto &c = clients_[vec[td::Random::fast(0, td::narrow_cast(vec.size() - 1))]]; + + td::actor::send_closure(c->client, &AdnlExtClient::send_query, std::move(name), std::move(data), timeout, + std::move(promise)); +} + +void AdnlExtMultiClientImpl::client_ready(td::uint32 idx, bool value) { + auto it = clients_.find(idx); + if (it == clients_.end()) { + return; + } + auto &c = it->second; + if (c->ready == value) { + return; + } + c->ready = value; + if (value) { + total_ready_++; + if (total_ready_ == 1) { + callback_->on_ready(); + } + } else { + total_ready_--; + if (total_ready_ == 0) { + callback_->on_stop_ready(); + } + } +} + +std::unique_ptr AdnlExtMultiClientImpl::make_callback(td::uint32 g) { + class Cb : public Callback { + public: + Cb(td::actor::ActorId id, td::uint32 idx) : id_(id), idx_(idx) { + } + + void on_ready() override { + td::actor::send_closure(id_, &AdnlExtMultiClientImpl::client_ready, idx_, true); + } + + void on_stop_ready() override { + td::actor::send_closure(id_, &AdnlExtMultiClientImpl::client_ready, idx_, false); + } + + private: + td::actor::ActorId id_; + td::uint32 idx_; + }; + return std::make_unique(actor_id(this), g); +} + +td::actor::ActorOwn AdnlExtMultiClient::create( + std::vector> ids, std::unique_ptr callback) { + return td::actor::create_actor("extmulticlient", std::move(ids), std::move(callback)); +} + } // namespace adnl } // namespace ton diff --git a/adnl/adnl-ext-client.h b/adnl/adnl-ext-client.h index babccced..bd60f202 100644 --- a/adnl/adnl-ext-client.h +++ b/adnl/adnl-ext-client.h @@ -43,6 +43,14 @@ class AdnlExtClient : public td::actor::Actor { std::unique_ptr callback); }; +class AdnlExtMultiClient : public AdnlExtClient { + public: + virtual void add_server(AdnlNodeIdFull dst, td::IPAddress dst_addr, td::Promise promise) = 0; + virtual void del_server(td::IPAddress dst_addr, td::Promise promise) = 0; + static td::actor::ActorOwn create(std::vector> ids, + std::unique_ptr callback); +}; + } // namespace adnl } // namespace ton diff --git a/adnl/adnl-ext-client.hpp b/adnl/adnl-ext-client.hpp index 95348135..d58247fc 100644 --- a/adnl/adnl-ext-client.hpp +++ b/adnl/adnl-ext-client.hpp @@ -140,6 +140,52 @@ class AdnlExtClientImpl : public AdnlExtClient { void try_stop(); }; +class AdnlExtMultiClientImpl : public AdnlExtMultiClient { + public: + AdnlExtMultiClientImpl(std::vector> ids, + std::unique_ptr callback) + : ids_(std::move(ids)), callback_(std::move(callback)) { + } + + void start_up() override; + + void add_server(AdnlNodeIdFull dst, td::IPAddress dst_addr, td::Promise promise) override; + void del_server(td::IPAddress dst_addr, td::Promise promise) override; + + void check_ready(td::Promise promise) override { + if (total_ready_ > 0) { + promise.set_value(td::Unit()); + } else { + promise.set_error(td::Status::Error(ErrorCode::notready, "conn not ready")); + } + } + void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + td::Promise promise) override; + + void client_ready(td::uint32 idx, bool value); + + private: + std::unique_ptr make_callback(td::uint32 g); + + struct Client { + Client(td::actor::ActorOwn client, AdnlNodeIdFull pubkey, td::IPAddress addr, td::uint32 generation) + : client(std::move(client)), pubkey(std::move(pubkey)), addr(addr), generation(generation), ready(false) { + } + td::actor::ActorOwn client; + AdnlNodeIdFull pubkey; + td::IPAddress addr; + td::uint32 generation; + bool ready = false; + }; + td::uint32 total_ready_ = 0; + + td::uint32 generation_ = 0; + std::map> clients_; + + std::vector> ids_; + std::unique_ptr callback_; +}; + } // namespace adnl } // namespace ton diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index a96434bd..481b79fb 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -139,8 +139,8 @@ ton::tl_object_ptr Config::tl() const { std::vector> val_vec; - ton::tl_object_ptr full_node_slave_ = nullptr; - std::vector> full_node_masters_; + std::vector> full_node_slaves_vec; + std::vector> full_node_masters_vec; std::vector> liteserver_vec; @@ -160,7 +160,7 @@ ton::tl_object_ptr Config::tl() const { } return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), - ton::PublicKeyHash::zero().tl(), std::move(full_node_slave_), std::move(full_node_masters_), + ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); } diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 27e5d985..dd2a3a61 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -25,15 +25,40 @@ namespace td { -Timer::Timer() : start_time_(Time::now()) { +Timer::Timer(bool is_paused) : is_paused_(is_paused) { + if (is_paused_) { + start_time_ = 0; + } else { + start_time_ = Time::now(); + } +} + +void Timer::pause() { + if (is_paused_) { + return; + } + elapsed_ += Time::now() - start_time_; + is_paused_ = true; +} + +void Timer::resume() { + if (!is_paused_) { + return; + } + start_time_ = Time::now(); + is_paused_ = false; } double Timer::elapsed() const { - return Time::now() - start_time_; + double res = elapsed_; + if (!is_paused_) { + res += Time::now() - start_time_; + } + return res; } StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) { - return string_builder << "in " << Time::now() - timer.start_time_; + return string_builder << format::as_time(timer.elapsed()); } PerfWarningTimer::PerfWarningTimer(string name, double max_duration) diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 1b84b7da..ed5afa74 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -24,14 +24,22 @@ namespace td { class Timer { public: - Timer(); + Timer() : Timer(false) { + } + explicit Timer(bool is_paused); + Timer(const Timer &other) = default; + Timer &operator=(const Timer &other) = default; double elapsed() const; + void pause(); + void resume(); private: friend StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer); + double elapsed_{0}; double start_time_; + bool is_paused_{false}; }; class PerfWarningTimer { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 073595ce..bd290a1a 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -322,6 +322,7 @@ tonNode.zeroStateIdExt workchain:int root_hash:int256 file_hash:int256 = tonNode tonNode.blockDescriptionEmpty = tonNode.BlockDescription; tonNode.blockDescription id:tonNode.blockIdExt = tonNode.BlockDescription; +tonNode.blocksDescription ids:(vector tonNode.blockIdExt) incomplete:Bool = tonNode.BlocksDescription; tonNode.preparedProofEmpty = tonNode.PreparedProof; tonNode.preparedProof = tonNode.PreparedProof; tonNode.preparedProofLink = tonNode.PreparedProof; @@ -352,20 +353,34 @@ tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId; ton.blockIdApprove root_cell_hash:int256 file_hash:int256 = ton.BlockId; +tonNode.dataList data:(vector bytes) = tonNode.DataList; + +tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull; +tonNode.dataFullEmpty = tonNode.DataFull; + ---functions--- tonNode.getNextBlockDescription prev_block:tonNode.blockIdExt = tonNode.BlockDescription; +tonNode.getNextBlocksDescription prev_block:tonNode.blockIdExt limit:int = tonNode.BlocksDescription; +tonNode.getPrevBlocksDescription next_block:tonNode.blockIdExt limit:int cutoff_seqno:int = tonNode.BlocksDescription; tonNode.prepareBlockProof block:tonNode.blockIdExt allow_partial:Bool = tonNode.PreparedProof; +tonNode.prepareBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:Bool = tonNode.PreparedProof; tonNode.prepareBlock block:tonNode.blockIdExt = tonNode.Prepared; +tonNode.prepareBlocks blocks:(vector tonNode.blockIdExt) = tonNode.Prepared; tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; tonNode.prepareZeroState block:tonNode.blockIdExt = tonNode.PreparedState; tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBlocks; +tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull; +tonNode.downloadBlockFull block:tonNode.blockIdExt = tonNode.DataFull; tonNode.downloadBlock block:tonNode.blockIdExt = tonNode.Data; +tonNode.downloadBlocks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data; tonNode.downloadZeroState block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data; +tonNode.downloadBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data; +tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.slave.sendExtMessage message:tonNode.externalMessage = True; @@ -490,7 +505,7 @@ engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNode engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) - validators:(vector engine.validator) fullnode:int256 fullnodeslave:engine.validator.fullNodeSlave + validators:(vector engine.validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) gc:engine.gc = engine.validator.Config; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 00dc97c4..b8e8300b 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 9535f056..356f263c 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -32,7 +32,7 @@ internal.transactionId lt:int64 hash:bytes = internal.TransactionId; raw.initialAccountState code:bytes data:bytes = raw.InitialAccountState; raw.accountState balance:int64 code:bytes data:bytes last_transaction_id:internal.transactionId sync_utime:int53 = raw.AccountState; -raw.message source:string destination:string value:int64 = raw.Message; +raw.message source:string destination:string value:int64 message:bytes = raw.Message; raw.transaction utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 in_msg:raw.message out_msgs:vector = raw.Transaction; raw.transactions transactions:vector previous_transaction_id:internal.transactionId = raw.Transactions; @@ -81,15 +81,15 @@ raw.getTransactions account_address:accountAddress from_transaction_id:internal. testWallet.init private_key:inputKey = Ok; testWallet.getAccountAddress initital_account_state:testWallet.initialAccountState = AccountAddress; testWallet.getAccountState account_address:accountAddress = testWallet.AccountState; -testWallet.sendGrams private_key:inputKey destination:accountAddress seqno:int32 amount:int64 = Ok; +testWallet.sendGrams private_key:inputKey destination:accountAddress seqno:int32 amount:int64 message:bytes = Ok; testGiver.getAccountState = testGiver.AccountState; testGiver.getAccountAddress = AccountAddress; -testGiver.sendGrams destination:accountAddress seqno:int32 amount:int64 = Ok; +testGiver.sendGrams destination:accountAddress seqno:int32 amount:int64 message:bytes = Ok; //generic.getAccountAddress initital_account_state:generic.InitialAccountState = AccountAddress; generic.getAccountState account_address:accountAddress = generic.AccountState; -generic.sendGrams private_key:inputKey source:accountAddress destination:accountAddress amount:int64 = Ok; +generic.sendGrams private_key:inputKey source:accountAddress destination:accountAddress amount:int64 message:bytes = Ok; onLiteServerQueryResult id:int64 bytes:bytes = Ok; onLiteServerQueryError id:int64 error:error = Ok; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 7d87ccac..40a4ccdc 100644 Binary files a/tl/generate/scheme/tonlib_api.tlo and b/tl/generate/scheme/tonlib_api.tlo differ diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index 6c2f11a4..9b9b81da 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -13,6 +13,7 @@ set(TONLIB_SOURCE tonlib/GenericAccount.cpp tonlib/KeyStorage.cpp tonlib/LastBlock.cpp + tonlib/LastBlockStorage.cpp tonlib/TestGiver.cpp tonlib/TestWallet.cpp tonlib/TonlibClient.cpp @@ -26,6 +27,7 @@ set(TONLIB_SOURCE tonlib/GenericAccount.h tonlib/KeyStorage.h tonlib/LastBlock.h + tonlib/LastBlockStorage.h tonlib/TestGiver.h tonlib/TestWallet.h tonlib/TonlibCallback.h diff --git a/tonlib/test/offline.cpp b/tonlib/test/offline.cpp index d5d86dba..e009adf7 100644 --- a/tonlib/test/offline.cpp +++ b/tonlib/test/offline.cpp @@ -127,8 +127,8 @@ TEST(Tonlib, TestGiver) { auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data; - auto res = GenericAccount::create_ext_message(TestGiver::address(), {}, - TestGiver::make_a_gift_message(0, 1000000000ll * 6666 / 1000, address)); + auto res = GenericAccount::create_ext_message( + TestGiver::address(), {}, TestGiver::make_a_gift_message(0, 1000000000ll * 6666 / 1000, "GIFT", address)); vm::CellSlice(vm::NoVm(), res).print_rec(std::cerr); CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == res->get_hash()); } diff --git a/tonlib/test/online.cpp b/tonlib/test/online.cpp index 8f5a73e9..716f9153 100644 --- a/tonlib/test/online.cpp +++ b/tonlib/test/online.cpp @@ -118,7 +118,7 @@ void transfer_grams(Client& client, std::string from, std::string to, td::int64 auto balance = get_balance(client, to); sync_send(client, tonlib_api::make_object( std::move(input_key), tonlib_api::make_object(from), - tonlib_api::make_object(to), amount)) + tonlib_api::make_object(to), amount, "GIFT")) .ensure(); while (balance == get_balance(client, to)) { client.receive(1); @@ -270,7 +270,7 @@ int main(int argc, char* argv[]) { { sync_send(client, make_object( make_object(wallet_addr.rserialize()), seqno, - 1000000000ll * 6666 / 1000)) + 1000000000ll * 6666 / 1000, "GIFT")) .ensure(); } @@ -307,9 +307,10 @@ int main(int argc, char* argv[]) { } { - sync_send(client, make_object( - create_input_key(), make_object(wallet_addr.rserialize()), - make_object(test_giver_address), 1000000000ll * 3333 / 1000)) + sync_send(client, + make_object( + create_input_key(), make_object(wallet_addr.rserialize()), + make_object(test_giver_address), 1000000000ll * 3333 / 1000, "GIFT")) .ensure(); } while (true) { diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 5633363d..0ec942ed 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -21,10 +21,10 @@ #include "tonlib/LastBlock.h" namespace tonlib { -void ExtClient::with_last_block(td::Promise promise) { +void ExtClient::with_last_block(td::Promise promise) { auto query_id = last_block_queries_.create(std::move(promise)); - td::Promise P = [query_id, self = this, - actor_id = td::actor::actor_id()](td::Result result) { + td::Promise P = [query_id, self = this, + actor_id = td::actor::actor_id()](td::Result result) { send_lambda(actor_id, [self, query_id, result = std::move(result)]() mutable { self->last_block_queries_.extract(query_id).set_result(std::move(result)); }); diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index 8f92ea9e..dca5cdfd 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -29,7 +29,7 @@ namespace tonlib { class LastBlock; -struct LastBlockInfo; +struct LastBlockState; struct ExtClientRef { td::actor::ActorId andl_ext_client_; td::actor::ActorId last_block_actor_; @@ -50,7 +50,7 @@ class ExtClient { return client_; } - void with_last_block(td::Promise promise); + void with_last_block(td::Promise promise); template void send_query(QueryT query, td::Promise promise) { @@ -75,7 +75,7 @@ class ExtClient { private: ExtClientRef client_; td::Container> queries_; - td::Container> last_block_queries_; + td::Container> last_block_queries_; void send_raw_query(td::BufferSlice query, td::Promise promise); }; diff --git a/tonlib/tonlib/GenericAccount.cpp b/tonlib/tonlib/GenericAccount.cpp index ca020994..330393c0 100644 --- a/tonlib/tonlib/GenericAccount.cpp +++ b/tonlib/tonlib/GenericAccount.cpp @@ -28,7 +28,7 @@ td::Ref GenericAccount::get_init_state(td::Ref code, td::Ref .finalize(); } block::StdAddress GenericAccount::get_address(ton::WorkchainId workchain_id, const td::Ref& init_state) { - return block::StdAddress(workchain_id, init_state->get_hash().bits()); + return block::StdAddress(workchain_id, init_state->get_hash().bits(), false); } td::Ref GenericAccount::create_ext_message(const block::StdAddress& address, td::Ref new_state, td::Ref body) { diff --git a/tonlib/tonlib/LastBlock.cpp b/tonlib/tonlib/LastBlock.cpp index f4754023..3593adc5 100644 --- a/tonlib/tonlib/LastBlock.cpp +++ b/tonlib/tonlib/LastBlock.cpp @@ -23,13 +23,23 @@ #include "lite-client/lite-client-common.h" namespace tonlib { -LastBlock::LastBlock(ExtClientRef client, State state, td::unique_ptr callback) + +td::StringBuilder& operator<<(td::StringBuilder& sb, const LastBlockState& state) { + return sb << td::tag("last_block", state.last_block_id.to_str()) + << td::tag("last_key_block", state.last_key_block_id.to_str()) << td::tag("utime", state.utime); +} + +LastBlock::LastBlock(ExtClientRef client, LastBlockState state, td::unique_ptr callback) : state_(std::move(state)), callback_(std::move(callback)) { client_.set_client(client); } -void LastBlock::get_last_block(td::Promise promise) { +void LastBlock::get_last_block(td::Promise promise) { if (promises_.empty()) { + total_sync_ = td::Timer(); + validate_ = td::Timer(true); + queries_ = 0; + LOG(INFO) << "Begin last block synchronization " << state_; do_get_last_block(); } promises_.push_back(std::move(promise)); @@ -41,6 +51,7 @@ void LastBlock::do_get_last_block() { //return; //liteServer.getBlockProof mode:# known_block:tonNode.blockIdExt target_block:mode.0?tonNode.blockIdExt = liteServer.PartialBlockProof; + queries_++; client_.send_query( ton::lite_api::liteServer_getBlockProof(0, create_tl_lite_block_id(state_.last_key_block_id), nullptr), [this, from = state_.last_key_block_id](auto r_block_proof) { @@ -51,21 +62,30 @@ void LastBlock::do_get_last_block() { td::Result LastBlock::process_block_proof( ton::BlockIdExt from, td::Result> r_block_proof) { + validate_.resume(); + SCOPE_EXIT { + validate_.pause(); + }; + TRY_RESULT(block_proof, std::move(r_block_proof)); - LOG(ERROR) << to_string(block_proof); + LOG(DEBUG) << "Got proof FROM\n" << to_string(block_proof->from_) << "TO\n" << to_string(block_proof->to_); TRY_RESULT(chain, liteclient::deserialize_proof_chain(std::move(block_proof))); if (chain->from != from) { return td::Status::Error(PSLICE() << "block proof chain starts from block " << chain->from.to_str() << ", not from requested block " << from.to_str()); } TRY_STATUS(chain->validate()); - update_mc_last_block(chain->to); + bool is_changed = false; + is_changed |= update_mc_last_block(chain->to); if (chain->has_key_block) { - update_mc_last_key_block(chain->key_blkid); + is_changed |= update_mc_last_key_block(chain->key_blkid); } if (chain->has_utime) { update_utime(chain->last_utime); } + if (is_changed) { + callback_->on_state_changed(state_); + } return chain->complete; } @@ -73,17 +93,20 @@ void LastBlock::on_block_proof( ton::BlockIdExt from, td::Result> r_block_proof) { auto r_is_ready = process_block_proof(from, std::move(r_block_proof)); + bool is_ready; if (r_is_ready.is_error()) { - LOG(WARNING) << "Failed liteServer_getBlockProof " << r_is_ready.error(); - return; + LOG(WARNING) << "Error during last block synchronization " << r_is_ready.error(); + is_ready = true; + } else { + is_ready = r_is_ready.move_as_ok(); } - auto is_ready = r_is_ready.move_as_ok(); if (is_ready) { + LOG(INFO) << "End last block synchronization " << state_ << "\n" + << " net queries: " << queries_ << "\n" + << " total: " << total_sync_ << " validation: " << validate_; for (auto& promise : promises_) { - LastBlockInfo res; - res.id = state_.last_block_id; - res.utime = state_.utime; - promise.set_value(std::move(res)); + auto state = state_; + promise.set_value(std::move(state)); } promises_.clear(); } else { @@ -101,10 +124,8 @@ void LastBlock::on_masterchain_info( LOG(WARNING) << "Failed liteServer_getMasterchainInfo " << r_info.error(); } for (auto& promise : promises_) { - LastBlockInfo res; - res.id = state_.last_block_id; - res.utime = state_.utime; - promise.set_value(std::move(res)); + auto state = state_; + promise.set_value(std::move(state)); } promises_.clear(); } @@ -131,25 +152,30 @@ void LastBlock::update_zero_state(ton::ZeroStateIdExt zero_state_id) { // One will have to restart ton client } -void LastBlock::update_mc_last_block(ton::BlockIdExt mc_block_id) { +bool LastBlock::update_mc_last_block(ton::BlockIdExt mc_block_id) { if (!mc_block_id.is_valid()) { LOG(ERROR) << "Ignore invalid masterchain block"; - return; + return false; } if (!state_.last_block_id.is_valid() || state_.last_block_id.id.seqno < mc_block_id.id.seqno) { state_.last_block_id = mc_block_id; LOG(INFO) << "Update masterchain block id: " << state_.last_block_id.to_str(); + return true; } + return false; } -void LastBlock::update_mc_last_key_block(ton::BlockIdExt mc_key_block_id) { + +bool LastBlock::update_mc_last_key_block(ton::BlockIdExt mc_key_block_id) { if (!mc_key_block_id.is_valid()) { LOG(ERROR) << "Ignore invalid masterchain block"; - return; + return false; } if (!state_.last_key_block_id.is_valid() || state_.last_key_block_id.id.seqno < mc_key_block_id.id.seqno) { state_.last_key_block_id = mc_key_block_id; LOG(INFO) << "Update masterchain key block id: " << state_.last_key_block_id.to_str(); + return true; } + return false; } void LastBlock::update_utime(td::int64 utime) { diff --git a/tonlib/tonlib/LastBlock.h b/tonlib/tonlib/LastBlock.h index 968f21bd..bb250d74 100644 --- a/tonlib/tonlib/LastBlock.h +++ b/tonlib/tonlib/LastBlock.h @@ -22,35 +22,114 @@ #include "tonlib/ExtClient.h" namespace tonlib { -struct LastBlockInfo { - ton::BlockIdExt id; +td::StringBuilder &operator<<(td::StringBuilder &sb, const LastBlockState &state); +template +void store(const td::BitArray &arr, StorerT &storer) { + storer.store_binary(arr); +} +template +void parse(td::BitArray &arr, ParserT &parser) { + arr = parser.template fetch_binary>(); +} + +template +void store(const ton::ZeroStateIdExt &zero_state_id, StorerT &storer) { + using td::store; + using tonlib::store; + store(zero_state_id.workchain, storer); + store(zero_state_id.root_hash, storer); + store(zero_state_id.file_hash, storer); +} +template +void parse(ton::ZeroStateIdExt &zero_state_id, ParserT &parser) { + using td::parse; + using tonlib::parse; + parse(zero_state_id.workchain, parser); + parse(zero_state_id.root_hash, parser); + parse(zero_state_id.file_hash, parser); +} +template +void store(const ton::BlockId &block_id, StorerT &storer) { + using td::store; + using tonlib::store; + store(block_id.workchain, storer); + store(block_id.shard, storer); + store(block_id.seqno, storer); +} +template +void parse(ton::BlockId &block_id, ParserT &parser) { + using td::parse; + using tonlib::parse; + parse(block_id.workchain, parser); + parse(block_id.shard, parser); + parse(block_id.seqno, parser); +} +template +void store(const ton::BlockIdExt &block_id, StorerT &storer) { + using td::store; + using tonlib::store; + store(block_id.id, storer); + store(block_id.root_hash, storer); + store(block_id.file_hash, storer); +} +template +void parse(ton::BlockIdExt &block_id, ParserT &parser) { + using td::parse; + using tonlib::parse; + parse(block_id.id, parser); + parse(block_id.root_hash, parser); + parse(block_id.file_hash, parser); +} +struct LastBlockState { + ton::ZeroStateIdExt zero_state_id; + ton::BlockIdExt last_key_block_id; + ton::BlockIdExt last_block_id; td::int64 utime{0}; + + template + void store(StorerT &storer) const { + using td::store; + using tonlib::store; + store(zero_state_id, storer); + store(last_key_block_id, storer); + store(last_block_id, storer); + store(utime, storer); + } + + template + void parse(ParserT &parser) { + using td::parse; + using tonlib::parse; + parse(zero_state_id, parser); + parse(last_key_block_id, parser); + parse(last_block_id, parser); + parse(utime, parser); + } }; + class LastBlock : public td::actor::Actor { public: - struct State { - ton::ZeroStateIdExt zero_state_id; - ton::BlockIdExt last_key_block_id; - ton::BlockIdExt last_block_id; - td::int64 utime{0}; - }; - class Callback { public: virtual ~Callback() { } - virtual void on_state_changes(State state) = 0; + virtual void on_state_changed(LastBlockState state) = 0; }; - explicit LastBlock(ExtClientRef client, State state, td::unique_ptr callback); - void get_last_block(td::Promise promise); + explicit LastBlock(ExtClientRef client, LastBlockState state, td::unique_ptr callback); + void get_last_block(td::Promise promise); private: ExtClient client_; - State state_; + LastBlockState state_; td::unique_ptr callback_; - std::vector> promises_; + // stats + td::Timer total_sync_; + td::Timer validate_; + td::uint32 queries_; + + std::vector> promises_; void do_get_last_block(); void on_masterchain_info(td::Result> r_info); @@ -62,8 +141,8 @@ class LastBlock : public td::actor::Actor { void update_zero_state(ton::ZeroStateIdExt zero_state_id); - void update_mc_last_block(ton::BlockIdExt mc_block_id); - void update_mc_last_key_block(ton::BlockIdExt mc_key_block_id); + bool update_mc_last_block(ton::BlockIdExt mc_block_id); + bool update_mc_last_key_block(ton::BlockIdExt mc_key_block_id); void update_utime(td::int64 utime); }; } // namespace tonlib diff --git a/tonlib/tonlib/LastBlockStorage.cpp b/tonlib/tonlib/LastBlockStorage.cpp new file mode 100644 index 00000000..fa979a3c --- /dev/null +++ b/tonlib/tonlib/LastBlockStorage.cpp @@ -0,0 +1,44 @@ +#include "LastBlockStorage.h" + +#include "td/utils/as.h" +#include "td/utils/filesystem.h" +#include "td/utils/port/path.h" +#include "td/utils/tl_helpers.h" + +namespace tonlib { + +td::Status LastBlockStorage::set_directory(std::string directory) { + TRY_RESULT(path, td::realpath(directory)); + TRY_RESULT(stat, td::stat(path)); + if (!stat.is_dir_) { + return td::Status::Error("not a directory"); + } + directory_ = std::move(path); + return td::Status::OK(); +} + +std::string LastBlockStorage::get_file_name(td::Slice name) { + return directory_ + TD_DIR_SLASH + td::buffer_to_hex(name) + ".blkstate"; +} + +td::Result LastBlockStorage::get_state(td::Slice name) { + TRY_RESULT(data, td::read_file(get_file_name(name))); + if (data.size() < 8) { + return td::Status::Error("too short"); + } + if (td::as(data.data()) != td::crc64(td::Slice(data).substr(8))) { + return td::Status::Error("crc64 mismatch"); + } + LastBlockState res; + TRY_STATUS(td::unserialize(res, td::Slice(data).substr(8))); + return res; +} + +void LastBlockStorage::save_state(td::Slice name, LastBlockState state) { + auto x = td::serialize(state); + std::string y(x.size() + 8, 0); + td::MutableSlice(y).substr(8).copy_from(x); + td::as(td::MutableSlice(y).data()) = td::crc64(x); + td::atomic_write_file(get_file_name(name), y); +} +} // namespace tonlib diff --git a/tonlib/tonlib/LastBlockStorage.h b/tonlib/tonlib/LastBlockStorage.h new file mode 100644 index 00000000..fa920ef6 --- /dev/null +++ b/tonlib/tonlib/LastBlockStorage.h @@ -0,0 +1,16 @@ +#pragma once + +#include "tonlib/LastBlock.h" + +namespace tonlib { +class LastBlockStorage { + public: + td::Status set_directory(std::string directory); + td::Result get_state(td::Slice name); + void save_state(td::Slice name, LastBlockState state); + + private: + std::string directory_; + std::string get_file_name(td::Slice name); +}; +} // namespace tonlib diff --git a/tonlib/tonlib/TestGiver.cpp b/tonlib/tonlib/TestGiver.cpp index c9ebbcc9..9ea58e54 100644 --- a/tonlib/tonlib/TestGiver.cpp +++ b/tonlib/tonlib/TestGiver.cpp @@ -32,8 +32,9 @@ vm::CellHash TestGiver::get_init_code_hash() { return vm::CellHash::from_slice(td::base64_decode("wDkZp0yR4xo+9+BnuAPfGVjBzK6FPzqdv2DwRq3z3KE=").move_as_ok()); } -td::Ref TestGiver::make_a_gift_message(td::uint32 seqno, td::uint64 gramms, +td::Ref TestGiver::make_a_gift_message(td::uint32 seqno, td::uint64 gramms, td::Slice message, const block::StdAddress& dest_address) { + CHECK(message.size() <= 128); td::BigInt256 dest_addr; dest_addr.import_bits(dest_address.addr.as_bitslice()); vm::CellBuilder cb; @@ -44,7 +45,7 @@ td::Ref TestGiver::make_a_gift_message(td::uint32 seqno, td::uint64 gr .store_int256(dest_addr, 256); block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(gramms)); - auto message_inner = cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes("GIFT").finalize(); + auto message_inner = cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes(message).finalize(); return vm::CellBuilder().store_long(seqno, 32).store_long(1, 8).store_ref(message_inner).finalize(); } } // namespace tonlib diff --git a/tonlib/tonlib/TestGiver.h b/tonlib/tonlib/TestGiver.h index 88ccd1b7..6859be8f 100644 --- a/tonlib/tonlib/TestGiver.h +++ b/tonlib/tonlib/TestGiver.h @@ -23,7 +23,7 @@ class TestGiver { public: static const block::StdAddress& address(); static vm::CellHash get_init_code_hash(); - static td::Ref make_a_gift_message(td::uint32 seqno, td::uint64 gramms, + static td::Ref make_a_gift_message(td::uint32 seqno, td::uint64 gramms, td::Slice message, const block::StdAddress& dest_address); }; } // namespace tonlib diff --git a/tonlib/tonlib/TestWallet.cpp b/tonlib/tonlib/TestWallet.cpp index bda32c17..c092bac2 100644 --- a/tonlib/tonlib/TestWallet.cpp +++ b/tonlib/tonlib/TestWallet.cpp @@ -38,7 +38,9 @@ td::Ref TestWallet::get_init_message(const td::Ed25519::PrivateKey& pr } td::Ref TestWallet::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 seqno, - td::int64 gramms, const block::StdAddress& dest_address) { + td::int64 gramms, td::Slice message, + const block::StdAddress& dest_address) { + CHECK(message.size() <= 128); td::BigInt256 dest_addr; dest_addr.import_bits(dest_address.addr.as_bitslice()); vm::CellBuilder cb; @@ -46,11 +48,11 @@ td::Ref TestWallet::make_a_gift_message(const td::Ed25519::PrivateKey& .store_long(dest_address.workchain, 8) .store_int256(dest_addr, 256); block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(gramms)); - auto message_inner = cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes("GIFT").finalize(); - auto message = vm::CellBuilder().store_long(seqno, 32).store_long(1, 8).store_ref(message_inner).finalize(); + auto message_inner = cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes(message).finalize(); + auto message_outer = vm::CellBuilder().store_long(seqno, 32).store_long(1, 8).store_ref(message_inner).finalize(); std::string seq_no(4, 0); - auto signature = private_key.sign(message->get_hash().as_slice()).move_as_ok(); - return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message)).finalize(); + auto signature = private_key.sign(message_outer->get_hash().as_slice()).move_as_ok(); + return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); } td::Ref TestWallet::get_init_code() { diff --git a/tonlib/tonlib/TestWallet.h b/tonlib/tonlib/TestWallet.h index f4de1f6a..921e3b3a 100644 --- a/tonlib/tonlib/TestWallet.h +++ b/tonlib/tonlib/TestWallet.h @@ -28,7 +28,8 @@ class TestWallet { static td::Ref get_init_state(const td::Ed25519::PublicKey& public_key); static td::Ref get_init_message(const td::Ed25519::PrivateKey& private_key); static td::Ref make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 seqno, - td::int64 gramms, const block::StdAddress& dest_address); + td::int64 gramms, td::Slice message, + const block::StdAddress& dest_address); static td::Ref get_init_code(); static vm::CellHash get_init_code_hash(); diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 7b48876f..ea0c6c81 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -41,8 +41,6 @@ #include "td/utils/tests.h" #include "td/utils/port/path.h" -namespace ton {} // namespace ton - namespace tonlib { template @@ -152,6 +150,11 @@ class GetTransactionHistory : public td::actor::Actor { } void start_up() override { + if (lt_ == 0) { + promise_.set_value(block::TransactionList::Info()); + stop(); + return; + } client_.send_query( ton::lite_api::liteServer_getTransactions( count_, ton::create_tl_object(address_.workchain, address_.addr), lt_, @@ -171,7 +174,7 @@ class GetRawAccountState : public td::actor::Actor { block::StdAddress address_; td::Promise promise_; ExtClient client_; - LastBlockInfo last_block_; + LastBlockState last_block_; void with_account_state(td::Result> r_account_state) { promise_.set_result(TRY_VM(do_with_account_state(std::move(r_account_state)))); @@ -182,7 +185,7 @@ class GetRawAccountState : public td::actor::Actor { td::Result> r_account_state) { TRY_RESULT(raw_account_state, std::move(r_account_state)); auto account_state = create_account_state(std::move(raw_account_state)); - TRY_RESULT(info, account_state.validate(last_block_.id, address_)); + TRY_RESULT(info, account_state.validate(last_block_.last_block_id, address_)); auto serialized_state = account_state.state.clone(); RawAccountState res; res.info = std::move(info); @@ -225,14 +228,14 @@ class GetRawAccountState : public td::actor::Actor { } void start_up() override { - client_.with_last_block([self = this](td::Result r_last_block) { + client_.with_last_block([self = this](td::Result r_last_block) { if (r_last_block.is_error()) { return self->check(r_last_block.move_as_error()); } self->last_block_ = r_last_block.move_as_ok(); self->client_.send_query( - ton::lite_api::liteServer_getAccountState(ton::create_tl_lite_block_id(self->last_block_.id), + ton::lite_api::liteServer_getAccountState(ton::create_tl_lite_block_id(self->last_block_.last_block_id), ton::create_tl_object( self->address_.workchain, self->address_.addr)), [self](auto r_state) { self->with_account_state(std::move(r_state)); }); @@ -307,24 +310,36 @@ void TonlibClient::init_ext_client() { td::make_unique(td::actor::actor_shared())); } } + +void TonlibClient::update_last_block_state(LastBlockState state) { + last_block_storage_.save_state("none", state); +} + void TonlibClient::init_last_block() { ref_cnt_++; class Callback : public LastBlock::Callback { public: Callback(td::actor::ActorShared client) : client_(std::move(client)) { } - void on_state_changes(LastBlock::State state) override { - //TODO + void on_state_changed(LastBlockState state) override { + send_closure(client_, &TonlibClient::update_last_block_state, std::move(state)); } private: td::actor::ActorShared client_; }; - LastBlock::State state; - //state.zero_state_id = ton::ZeroStateIdExt(config_.zero_state_id.id.workchain, config_.zero_state_id.root_hash, - //config_.zero_state_id.file_hash), - state.last_block_id = config_.zero_state_id; - state.last_key_block_id = config_.zero_state_id; + LastBlockState state; + + auto r_state = last_block_storage_.get_state("none"); + if (r_state.is_error()) { + LOG(WARNING) << "Unknown LastBlockState: " << r_state.error(); + state.zero_state_id = ton::ZeroStateIdExt(config_.zero_state_id.id.workchain, config_.zero_state_id.root_hash, + config_.zero_state_id.file_hash), + state.last_block_id = config_.zero_state_id; + state.last_key_block_id = config_.zero_state_id; + } else { + state = r_state.move_as_ok(); + } raw_last_block_ = td::actor::create_actor("LastBlock", get_client_ref(), std::move(state), td::make_unique(td::actor::actor_shared(this))); @@ -469,6 +484,7 @@ td::Status TonlibClient::do_request(const tonlib_api::init& request, return td::Status::Error(400, "Field options must not be empty"); } TRY_STATUS(key_storage_.set_directory(request.options_->keystore_directory_)); + TRY_STATUS(last_block_storage_.set_directory(request.options_->keystore_directory_)); use_callbacks_for_network_ = request.options_->use_callbacks_for_network_; if (!request.options_->config_.empty()) { TRY_STATUS(set_config(std::move(request.options_->config_))); @@ -570,10 +586,25 @@ td::Result> to_raw_message_or_th if (!tlb::csr_unpack(message.info, msg_info)) { return td::Status::Error("Failed to unpack CommonMsgInfo::int_msg_info"); } + TRY_RESULT(balance, to_balance(msg_info.value)); TRY_RESULT(src, to_std_address(msg_info.src)); TRY_RESULT(dest, to_std_address(msg_info.dest)); - return tonlib_api::make_object(std::move(src), std::move(dest), balance); + td::Ref body; + if (message.body->prefetch_long(1) == 0) { + body = std::move(message.body); + body.write().advance(1); + } else { + body = vm::load_cell_slice_ref(message.body->prefetch_ref()); + } + std::string body_message; + if (body->size() % 8 == 0) { + body_message = std::string(body->size() / 8, 0); + body->prefetch_bytes(td::MutableSlice(body_message).ubegin(), body->size() / 8); + } + + return tonlib_api::make_object(std::move(src), std::move(dest), balance, + std::move(body_message)); } case block::gen::CommonMsgInfo::ext_in_msg_info: { block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; @@ -581,7 +612,7 @@ td::Result> to_raw_message_or_th return td::Status::Error("Failed to unpack CommonMsgInfo::ext_in_msg_info"); } TRY_RESULT(dest, to_std_address(msg_info.dest)); - return tonlib_api::make_object("", std::move(dest), 0); + return tonlib_api::make_object("", std::move(dest), 0, ""); } case block::gen::CommonMsgInfo::ext_out_msg_info: { block::gen::CommonMsgInfo::Record_ext_out_msg_info msg_info; @@ -589,7 +620,7 @@ td::Result> to_raw_message_or_th return td::Status::Error("Failed to unpack CommonMsgInfo::ext_out_msg_info"); } TRY_RESULT(src, to_std_address(msg_info.src)); - return tonlib_api::make_object(std::move(src), "", 0); + return tonlib_api::make_object(std::move(src), "", 0, ""); } } @@ -626,7 +657,7 @@ td::Result> to_raw_transacti if (is_just == trans.r1.in_msg->fetch_long_eof) { return td::Status::Error("Failed to parse long"); } - if (is_just == 1) { + if (is_just == -1) { auto msg = trans.r1.in_msg->prefetch_ref(); TRY_RESULT(in_msg_copy, to_raw_message(trans.r1.in_msg->prefetch_ref())); in_msg = std::move(in_msg_copy); @@ -702,7 +733,7 @@ td::Result> to_generic_ RawAccountState&& raw_state) { if (raw_state.code.is_null()) { return tonlib_api::make_object( - tonlib_api::make_object(raw_state.balance, empty_transaction_id(), + tonlib_api::make_object(raw_state.balance, to_transaction_id(raw_state.info), raw_state.sync_utime)); } @@ -827,21 +858,24 @@ td::Status TonlibClient::do_request(const tonlib_api::testWallet_sendGrams& requ if (!request.private_key_) { return td::Status::Error(400, "Field private_key must not be empty"); } + if (request.message_.size() > 128) { + return td::Status::Error(400, "Message is too long"); + } TRY_RESULT(account_address, block::StdAddress::parse(request.destination_->account_address_)); account_address.bounceable = false; TRY_RESULT(input_key, from_tonlib(*request.private_key_)); auto address = GenericAccount::get_address( 0 /*zerochain*/, TestWallet::get_init_state(td::Ed25519::PublicKey(input_key.key.public_key.copy()))); TRY_RESULT(private_key, key_storage_.load_private_key(std::move(input_key))); - return do_request( - tonlib_api::raw_sendMessage(tonlib_api::make_object(address.rserialize()), "", - vm::std_boc_serialize(TestWallet::make_a_gift_message( - td::Ed25519::PrivateKey(std::move(private_key.private_key)), - request.seqno_, request.amount_, account_address)) - .move_as_ok() - .as_slice() - .str()), - std::move(promise)); + return do_request(tonlib_api::raw_sendMessage( + tonlib_api::make_object(address.rserialize()), "", + vm::std_boc_serialize(TestWallet::make_a_gift_message( + td::Ed25519::PrivateKey(std::move(private_key.private_key)), + request.seqno_, request.amount_, request.message_, account_address)) + .move_as_ok() + .as_slice() + .str()), + std::move(promise)); } td::Status TonlibClient::do_request(tonlib_api::testWallet_getAccountState& request, @@ -868,16 +902,19 @@ td::Status TonlibClient::do_request(const tonlib_api::testGiver_sendGrams& reque if (!request.destination_) { return td::Status::Error(400, "Field destination must not be empty"); } + if (request.message_.size() > 128) { + return td::Status::Error(400, "Message is too long"); + } TRY_RESULT(account_address, block::StdAddress::parse(request.destination_->account_address_)); account_address.bounceable = false; - return do_request( - tonlib_api::raw_sendMessage( - tonlib_api::make_object(TestGiver::address().rserialize()), "", - vm::std_boc_serialize(TestGiver::make_a_gift_message(request.seqno_, request.amount_, account_address)) - .move_as_ok() - .as_slice() - .str()), - std::move(promise)); + return do_request(tonlib_api::raw_sendMessage( + tonlib_api::make_object(TestGiver::address().rserialize()), "", + vm::std_boc_serialize(TestGiver::make_a_gift_message(request.seqno_, request.amount_, + request.message_, account_address)) + .move_as_ok() + .as_slice() + .str()), + std::move(promise)); } td::Status TonlibClient::do_request(const tonlib_api::testGiver_getAccountState& request, @@ -920,7 +957,7 @@ td::Status TonlibClient::do_request(tonlib_api::generic_sendGrams& request, "GetAccountState", client_.get_client(), std::move(account_address), [promise = std::move(promise), self = this, actor_id = td::actor::actor_id(), private_key = std::move(request.private_key_), destination = std::move(request.destination_), - amount = request.amount_](td::Result r_state) mutable { + amount = request.amount_, message = std::move(request.message_)](td::Result r_state) mutable { if (r_state.is_error()) { return promise.set_error(r_state.move_as_error()); } @@ -930,40 +967,41 @@ td::Status TonlibClient::do_request(tonlib_api::generic_sendGrams& request, } auto state = rr_state.move_as_ok(); - downcast_call(*state, - td::overloaded( - [&](tonlib_api::generic_accountStateTestGiver& test_giver_state) { - send_lambda(actor_id, [promise = std::move(promise), self, - query = tonlib_api::testGiver_sendGrams( - std::move(destination), test_giver_state.account_state_->seqno_, - amount)]() mutable { - LOG(INFO) << "Send " << to_string(query); - auto status = self->do_request(query, std::move(promise)); - if (status.is_error()) { - CHECK(promise); - promise.set_error(std::move(status)); - } - }); - return; - }, - [&](tonlib_api::generic_accountStateTestWallet& test_wallet_state) { - send_lambda(actor_id, [promise = std::move(promise), self, - query = tonlib_api::testWallet_sendGrams( - std::move(private_key), std::move(destination), - test_wallet_state.account_state_->seqno_, amount)]() mutable { - auto status = self->do_request(query, std::move(promise)); - if (status.is_error()) { - CHECK(promise); - promise.set_error(std::move(status)); - } - }); - }, - [&](tonlib_api::generic_accountStateUninited&) { - promise.set_error(td::Status::Error(400, "Account is not inited")); - }, - [&](tonlib_api::generic_accountStateRaw&) { - promise.set_error(td::Status::Error(400, "Unknown account type")); - })); + downcast_call(*state, td::overloaded( + [&](tonlib_api::generic_accountStateTestGiver& test_giver_state) { + send_lambda(actor_id, + [promise = std::move(promise), self, + query = tonlib_api::testGiver_sendGrams( + std::move(destination), test_giver_state.account_state_->seqno_, + amount, std::move(message))]() mutable { + LOG(INFO) << "Send " << to_string(query); + auto status = self->do_request(query, std::move(promise)); + if (status.is_error()) { + CHECK(promise); + promise.set_error(std::move(status)); + } + }); + return; + }, + [&](tonlib_api::generic_accountStateTestWallet& test_wallet_state) { + send_lambda(actor_id, [promise = std::move(promise), self, + query = tonlib_api::testWallet_sendGrams( + std::move(private_key), std::move(destination), + test_wallet_state.account_state_->seqno_, amount, + std::move(message))]() mutable { + auto status = self->do_request(query, std::move(promise)); + if (status.is_error()) { + CHECK(promise); + promise.set_error(std::move(status)); + } + }); + }, + [&](tonlib_api::generic_accountStateUninited&) { + promise.set_error(td::Status::Error(400, "Account is not inited")); + }, + [&](tonlib_api::generic_accountStateRaw&) { + promise.set_error(td::Status::Error(400, "Unknown account type")); + })); }) .release(); return td::Status::OK(); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 0e36a289..a45fd6fd 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -24,6 +24,7 @@ #include "tonlib/ExtClient.h" #include "tonlib/ExtClientOutbound.h" #include "tonlib/KeyStorage.h" +#include "tonlib/LastBlockStorage.h" #include "td/actor/actor.h" @@ -50,6 +51,7 @@ class TonlibClient : public td::actor::Actor { // KeyStorage KeyStorage key_storage_; + LastBlockStorage last_block_storage_; // network td::actor::ActorOwn raw_client_; @@ -73,6 +75,7 @@ class TonlibClient : public td::actor::Actor { } } + void update_last_block_state(LastBlockState state); void on_result(td::uint64 id, object_ptr response); static bool is_static_request(td::int32 id); static bool is_uninited_request(td::int32 id); diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index cdbec51b..f6ce8883 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -6,6 +6,7 @@ #include "td/utils/port/signals.h" #include "td/utils/port/path.h" #include "td/utils/Random.h" +#include "td/utils/as.h" #include "terminal/terminal.h" @@ -327,6 +328,8 @@ class TonlibCli : public td::actor::Actor { << "\n"; for (size_t i = 0; i < keys_.size(); i++) { td::TerminalIO::out() << " #" << i << ": " << keys_[i].public_key << "\n"; + td::TerminalIO::out() << " " << to_account_address(PSLICE() << i, false).move_as_ok().address->account_address_ + << "\n"; } } @@ -598,13 +601,21 @@ class TonlibCli : public td::actor::Actor { } void transfer(Address from, Address to, td::uint64 grams, td::Slice password) { + td::TerminalIO::out() << "Enter message (could be empty)"; + cont_ = [this, from = std::move(from), to = std::move(to), grams, + password = password.str()](td::Slice message) mutable { + this->transfer(std::move(from), std::move(to), grams, password, message); + }; + return; + } + void transfer(Address from, Address to, td::uint64 grams, td::Slice password, td::Slice message) { using tonlib_api::make_object; auto key = !from.secret.empty() ? make_object( make_object(from.public_key, from.secret.copy()), td::SecureString(password)) : nullptr; send_query(make_object(std::move(key), std::move(from.address), - std::move(to.address), grams), + std::move(to.address), grams, message.str()), [](auto r_res) { if (r_res.is_error()) { td::TerminalIO::out() << "Can't get state: " << r_res.error() << "\n"; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index a118fcd4..d679288f 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -121,12 +121,10 @@ Config::Config(ton::ton_api::engine_validator_config &config) { } config_add_full_node_adnl_id(ton::PublicKeyHash{config.fullnode_}).ensure(); - if (config.fullnodeslave_) { + for (auto &s : config.fullnodeslaves_) { td::IPAddress ip; - ip.init_ipv4_port(td::IPAddress::ipv4_to_str(config.fullnodeslave_->ip_), - static_cast(config.fullnodeslave_->port_)) - .ensure(); - config_add_full_node_slave(ip, ton::PublicKey{config.fullnodeslave_->adnl_}).ensure(); + ip.init_ipv4_port(td::IPAddress::ipv4_to_str(s->ip_), static_cast(s->port_)).ensure(); + config_add_full_node_slave(ip, ton::PublicKey{s->adnl_}).ensure(); } for (auto &s : config.fullnodemasters_) { @@ -192,10 +190,10 @@ ton::tl_object_ptr Config::tl() const { val.first.tl(), std::move(temp_vec), std::move(adnl_val_vec), val.second.election_date, val.second.expire_at)); } - ton::tl_object_ptr full_node_slave = nullptr; - if (!full_node_slave_adnl_id.empty()) { - full_node_slave = ton::create_tl_object( - full_node_slave_addr.get_ipv4(), full_node_slave_addr.get_port(), full_node_slave_adnl_id.tl()); + std::vector> full_node_slaves_vec; + for (auto &x : full_node_slaves) { + full_node_slaves_vec.push_back(ton::create_tl_object( + x.addr.get_ipv4(), x.addr.get_port(), x.key.tl())); } std::vector> full_node_masters_vec; for (auto &x : full_node_masters) { @@ -224,8 +222,8 @@ ton::tl_object_ptr Config::tl() const { } return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), full_node.tl(), - std::move(full_node_slave), std::move(full_node_masters_vec), std::move(liteserver_vec), std::move(control_vec), - std::move(gc_vec)); + std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), + std::move(control_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -387,11 +385,16 @@ td::Result Config::config_add_full_node_adnl_id(ton::PublicKeyHash id) { } td::Result Config::config_add_full_node_slave(td::IPAddress addr, ton::PublicKey id) { - if (full_node_slave_adnl_id == id && addr == full_node_slave_addr) { - return false; + for (auto &s : full_node_slaves) { + if (s.addr == addr) { + if (s.key == id) { + return true; + } else { + return td::Status::Error(ton::ErrorCode::error, "duplicate slave ip"); + } + } } - full_node_slave_adnl_id = id; - full_node_slave_addr = addr; + full_node_slaves.push_back(FullNodeSlave{id, addr}); return true; } @@ -1462,11 +1465,15 @@ void ValidatorEngine::started_validator() { } void ValidatorEngine::start_full_node() { - if (!config_.full_node.is_zero() || !config_.full_node_slave_adnl_id.empty()) { + if (!config_.full_node.is_zero() || config_.full_node_slaves.size() > 0) { auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; auto short_id = pk.compute_short_id(); td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(pk), true, [](td::Unit) {}); - if (!config_.full_node_slave_adnl_id.empty()) { + if (config_.full_node_slaves.size() > 0) { + std::vector> vec; + for (auto &x : config_.full_node_slaves) { + vec.emplace_back(ton::adnl::AdnlNodeIdFull{x.key}, x.addr); + } class Cb : public ton::adnl::AdnlExtClient::Callback { public: void on_ready() override { @@ -1474,8 +1481,7 @@ void ValidatorEngine::start_full_node() { void on_stop_ready() override { } }; - full_node_client_ = ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{config_.full_node_slave_adnl_id}, - config_.full_node_slave_addr, std::make_unique()); + full_node_client_ = ton::adnl::AdnlExtMultiClient::create(std::move(vec), std::make_unique()); } full_node_ = ton::validator::fullnode::FullNode::create( short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index b0eae3dc..e67c5658 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -69,6 +69,10 @@ struct Config { ton::PublicKeyHash key; std::map clients; }; + struct FullNodeSlave { + ton::PublicKey key; + td::IPAddress addr; + }; std::map keys_refcnt; td::uint16 out_port; @@ -77,8 +81,7 @@ struct Config { std::set dht_ids; std::map validators; ton::PublicKeyHash full_node = ton::PublicKeyHash::zero(); - td::IPAddress full_node_slave_addr; - ton::PublicKey full_node_slave_adnl_id; + std::vector full_node_slaves; std::map full_node_masters; std::map liteservers; std::map controls; diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 4bc908d8..e66b4bfe 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -117,6 +117,8 @@ set(FULL_NODE_SOURCE net/download-block.hpp net/download-block.cpp + net/download-block-new.hpp + net/download-block-new.cpp net/download-next-block.hpp net/download-next-block.cpp net/download-state.hpp diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index 93543fe6..bade0ab8 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -18,6 +18,7 @@ */ #include "td/utils/SharedSlice.h" #include "full-node-master.hpp" +#include "full-node-shard-queries.hpp" #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" @@ -93,6 +94,20 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo create_block_id(query.block_), false, std::move(P)); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadBlockFull &query, + td::Promise promise) { + td::actor::create_actor("sender", ton::create_block_id(query.block_), false, validator_manager_, + std::move(promise)) + .release(); +} + +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadNextBlockFull &query, + td::Promise promise) { + td::actor::create_actor("sender", ton::create_block_id(query.prev_block_), true, validator_manager_, + std::move(promise)) + .release(); +} + void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareBlockProof &query, td::Promise promise) { if (query.block_->seqno_ == 0) { diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index 83bd88f4..0147ddca 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -46,6 +46,10 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadBlock &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadBlockFull &query, + td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadNextBlockFull &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareZeroState &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, diff --git a/validator/full-node-shard-queries.hpp b/validator/full-node-shard-queries.hpp new file mode 100644 index 00000000..7743d42f --- /dev/null +++ b/validator/full-node-shard-queries.hpp @@ -0,0 +1,114 @@ +#pragma once + +#include "validator/validator.h" +#include "ton/ton-tl.hpp" + +namespace ton { + +namespace validator { + +namespace fullnode { + +class BlockFullSender : public td::actor::Actor { + public: + BlockFullSender(BlockIdExt block_id, bool next, td::actor::ActorId manager, + td::Promise promise) + : block_id_(block_id), next_(next), manager_(manager), promise_(std::move(promise)) { + } + void abort_query(td::Status error) { + promise_.set_value(create_serialize_tl_object()); + stop(); + } + void finish_query() { + promise_.set_value(create_serialize_tl_object( + create_tl_block_id(block_id_), std::move(proof_), std::move(data_), is_proof_link_)); + stop(); + } + void start_up() override { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BlockFullSender::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &BlockFullSender::got_block_handle, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_handle, block_id_, false, std::move(P)); + } + + void got_block_handle(BlockHandle handle) { + if (next_) { + if (!handle->inited_next_left()) { + return abort_query(td::Status::Error(ErrorCode::notready, "next not known")); + } + next_ = false; + block_id_ = handle->one_next(true); + start_up(); + return; + } + if (!handle->received() || (!handle->inited_proof() && !handle->inited_proof_link()) || handle->deleted()) { + return abort_query(td::Status::Error(ErrorCode::notready, "not in db")); + } + handle_ = std::move(handle); + is_proof_link_ = !handle_->inited_proof(); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BlockFullSender::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &BlockFullSender::got_block_data, R.move_as_ok()->data()); + } + }); + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_data_from_db, handle_, std::move(P)); + + if (!is_proof_link_) { + auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BlockFullSender::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &BlockFullSender::got_block_proof, R.move_as_ok()->data()); + } + }); + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_proof_from_db, handle_, std::move(Q)); + } else { + auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BlockFullSender::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &BlockFullSender::got_block_proof, R.move_as_ok()->data()); + } + }); + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_proof_link_from_db, handle_, + std::move(Q)); + } + } + + void got_block_data(td::BufferSlice data) { + data_ = std::move(data); + if (!proof_.empty()) { + finish_query(); + } + } + + void got_block_proof(td::BufferSlice data) { + proof_ = std::move(data); + if (!data_.empty()) { + finish_query(); + } + } + + private: + BlockIdExt block_id_; + bool next_; + BlockHandle handle_; + bool is_proof_link_; + td::BufferSlice proof_; + td::BufferSlice data_; + td::actor::ActorId manager_; + td::Promise promise_; +}; + +} // namespace fullnode + +} // namespace validator + +} // namespace ton diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 912dc1c1..6e0f1afd 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -18,13 +18,16 @@ */ #include "td/utils/SharedSlice.h" #include "full-node-shard.hpp" +#include "full-node-shard-queries.hpp" #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" +#include "ton/ton-io.hpp" #include "adnl/utils.hpp" -#include "net/download-next-block.hpp" +#include "net/download-block-new.hpp" #include "net/download-block.hpp" +#include "net/download-next-block.hpp" #include "net/download-state.hpp" #include "net/download-proof.hpp" #include "net/get-next-key-blocks.hpp" @@ -77,15 +80,22 @@ void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise("downloadnext", adnl_id_, overlay_id_, handle_, download_next_priority(), - timeout, validator_manager_, rldp_, overlays_, adnl_, client_, - std::move(promise)) - .release(); + if (use_new_download()) { + td::actor::create_actor("downloadnext", adnl_id_, overlay_id_, handle_->id(), + ton::adnl::AdnlNodeIdShort::zero(), download_next_priority(), timeout, + validator_manager_, rldp_, overlays_, adnl_, client_, std::move(promise)) + .release(); + } else { + td::actor::create_actor("downloadnext", adnl_id_, overlay_id_, handle_, download_next_priority(), + timeout, validator_manager_, rldp_, overlays_, adnl_, client_, + std::move(promise)) + .release(); + } } void FullNodeShardImpl::got_next_block(td::Result R) { if (R.is_error()) { - if (R.error().code() == ErrorCode::timeout) { + if (R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) { get_next_block(); return; } @@ -199,6 +209,20 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod create_block_id(query.block_), false, std::move(P)); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadBlockFull &query, + td::Promise promise) { + td::actor::create_actor("sender", ton::create_block_id(query.block_), false, validator_manager_, + std::move(promise)) + .release(); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadNextBlockFull &query, + td::Promise promise) { + td::actor::create_actor("sender", ton::create_block_id(query.prev_block_), true, validator_manager_, + std::move(promise)) + .release(); +} + void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareBlockProof &query, td::Promise promise) { if (query.block_->seqno_ == 0) { @@ -520,10 +544,17 @@ void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - td::actor::create_actor("downloadreq", id, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), - priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, - std::move(promise)) - .release(); + if (use_new_download()) { + td::actor::create_actor("downloadreq", id, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), + priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, + std::move(promise)) + .release(); + } else { + td::actor::create_actor("downloadreq", id, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), + priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, + std::move(promise)) + .release(); + } } void FullNodeShardImpl::download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 20e96a3d..cf66da20 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -68,6 +68,10 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadBlock &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadBlockFull &query, + td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadNextBlockFull &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareZeroState &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, @@ -126,6 +130,10 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId client); private: + bool use_new_download() const { + return false; + } + ShardIdFull shard_; BlockHandle handle_; td::Promise promise_; diff --git a/validator/manager.cpp b/validator/manager.cpp index d93e4da3..8a0c79f6 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1665,7 +1665,21 @@ void ValidatorManagerImpl::allow_persistent_state_file_gc(BlockIdExt block_id, B promise.set_result(false); return; } - promise.set_result(masterchain_block_id.id.seqno + (1 << 17) < gc_masterchain_handle_->id().id.seqno); + if (masterchain_block_id.seqno() == 0) { + promise.set_result(false); + return; + } + if (masterchain_block_id.seqno() >= gc_masterchain_handle_->id().seqno()) { + promise.set_result(false); + return; + } + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + auto handle = R.move_as_ok(); + CHECK(handle->is_key_block()); + promise.set_result(ValidatorManager::persistent_state_ttl(handle->unix_time()) < td::Clocks::system()); + }); + get_block_handle(masterchain_block_id, false, std::move(P)); } void ValidatorManagerImpl::allow_archive(BlockIdExt block_id, td::Promise promise) { @@ -1814,6 +1828,16 @@ void ValidatorManagerImpl::state_serializer_update(BlockSeqno seqno) { void ValidatorManagerImpl::alarm() { try_advance_gc_masterchain_block(); alarm_timestamp() = td::Timestamp::in(1.0); + if (log_status_at_.is_in_past()) { + if (last_masterchain_block_handle_) { + LOG(INFO) << "STATUS: last_masterchain_block_ago=" + << td::format::as_time(td::Clocks::system() - last_masterchain_block_handle_->unix_time()) + << " last_known_key_block_ago=" + << td::format::as_time(td::Clocks::system() - last_known_key_block_handle_->unix_time()); + } + log_status_at_ = td::Timestamp::in(60.0); + } + alarm_timestamp().relax(log_status_at_); if (resend_shard_blocks_at_ && resend_shard_blocks_at_.is_in_past()) { resend_shard_blocks_at_ = td::Timestamp::never(); for (auto &B : out_shard_blocks_) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 930d6505..2b88696a 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -506,6 +506,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Timestamp resend_shard_blocks_at_; td::Timestamp check_waiters_at_; td::Timestamp check_shard_clients_; + td::Timestamp log_status_at_; void alarm() override; std::map> out_shard_blocks_; diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp new file mode 100644 index 00000000..6ce7c60e --- /dev/null +++ b/validator/net/download-block-new.cpp @@ -0,0 +1,283 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#include "download-block-new.hpp" +#include "ton/ton-tl.hpp" +#include "adnl/utils.hpp" +#include "ton/ton-shard.h" +#include "td/utils/overloaded.h" +#include "ton/ton-io.hpp" +#include "validator/full-node.h" + +namespace ton { + +namespace validator { + +namespace fullnode { + +DownloadBlockNew::DownloadBlockNew(BlockIdExt block_id, adnl::AdnlNodeIdShort local_id, + overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, + td::uint32 priority, td::Timestamp timeout, + td::actor::ActorId validator_manager, + td::actor::ActorId rldp, td::actor::ActorId overlays, + td::actor::ActorId adnl, td::actor::ActorId client, + td::Promise promise) + : block_id_(block_id) + , local_id_(local_id) + , overlay_id_(overlay_id) + , download_from_(download_from) + , priority_(priority) + , timeout_(timeout) + , validator_manager_(validator_manager) + , rldp_(rldp) + , overlays_(overlays) + , adnl_(adnl) + , client_(client) + , promise_(std::move(promise)) + , block_{block_id_, td::BufferSlice()} + , allow_partial_proof_{!block_id_.is_masterchain()} { +} + +DownloadBlockNew::DownloadBlockNew(adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, + BlockIdExt prev_id, adnl::AdnlNodeIdShort download_from, td::uint32 priority, + td::Timestamp timeout, + td::actor::ActorId validator_manager, + td::actor::ActorId rldp, td::actor::ActorId overlays, + td::actor::ActorId adnl, td::actor::ActorId client, + td::Promise promise) + : local_id_(local_id) + , overlay_id_(overlay_id) + , prev_id_(prev_id) + , download_from_(download_from) + , priority_(priority) + , timeout_(timeout) + , validator_manager_(validator_manager) + , rldp_(rldp) + , overlays_(overlays) + , adnl_(adnl) + , client_(client) + , promise_(std::move(promise)) + , block_{BlockIdExt{}, td::BufferSlice()} { +} + +void DownloadBlockNew::abort_query(td::Status reason) { + if (promise_) { + if (reason.code() == ErrorCode::notready || reason.code() == ErrorCode::timeout) { + VLOG(FULL_NODE_DEBUG) << "failed to download block " << block_id_ << "from " << download_from_ << ": " << reason; + } else { + VLOG(FULL_NODE_NOTICE) << "failed to download block " << block_id_ << " from " << download_from_ << ": " + << reason; + } + promise_.set_error(std::move(reason)); + } + stop(); +} + +void DownloadBlockNew::alarm() { + abort_query(td::Status::Error(ErrorCode::timeout, "timeout")); +} + +void DownloadBlockNew::finish_query() { + if (promise_) { + promise_.set_value(std::move(block_)); + } + stop(); +} + +void DownloadBlockNew::start_up() { + alarm_timestamp() = timeout_; + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &DownloadBlockNew::got_block_handle, R.move_as_ok()); + }); + + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_handle, + block_id_.is_valid() ? block_id_ : prev_id_, true, std::move(P)); +} + +void DownloadBlockNew::got_block_handle(BlockHandle handle) { + handle_ = std::move(handle); + + if (!block_id_.is_valid()) { + CHECK(prev_id_.is_valid()); + if (handle_->inited_next_left()) { + block_id_ = handle_->one_next(true); + block_.id = block_id_; + handle_ = nullptr; + start_up(); + return; + } + } + + if (block_id_.is_valid() && + (handle_->inited_proof() || (handle_->inited_proof_link() && allow_partial_proof_) || skip_proof_) && + handle_->received()) { + CHECK(block_.id == block_id_); + CHECK(handle_->id() == block_id_); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, + R.move_as_error_prefix("failed to get from db: ")); + } else { + td::actor::send_closure(SelfId, &DownloadBlockNew::got_data_from_db, R.move_as_ok()->data()); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_data_from_db, handle_, + std::move(P)); + return; + } + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, + R.move_as_error_prefix("failed to get download token: ")); + } else { + td::actor::send_closure(SelfId, &DownloadBlockNew::got_download_token, R.move_as_ok()); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_download_token, 1, priority_, timeout_, + std::move(P)); +} + +void DownloadBlockNew::got_download_token(std::unique_ptr token) { + token_ = std::move(token); + + if (download_from_.is_zero() && client_.empty()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, R.move_as_error()); + } else { + auto vec = R.move_as_ok(); + if (vec.size() == 0) { + td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, + td::Status::Error(ErrorCode::notready, "no nodes")); + } else { + td::actor::send_closure(SelfId, &DownloadBlockNew::got_node_to_download, vec[0]); + } + } + }); + + td::actor::send_closure(overlays_, &overlay::Overlays::get_overlay_random_peers, local_id_, overlay_id_, 1, + std::move(P)); + } else { + got_node_to_download(download_from_); + } +} + +void DownloadBlockNew::got_node_to_download(adnl::AdnlNodeIdShort node) { + download_from_ = node; + + VLOG(FULL_NODE_DEBUG) << "downloading proof for " << block_id_; + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadBlockNew::got_data, R.move_as_ok()); + } + }); + + td::BufferSlice q; + if (block_id_.is_valid()) { + q = create_serialize_tl_object(create_tl_block_id(block_id_)); + } else { + q = create_serialize_tl_object(create_tl_block_id(prev_id_)); + } + if (client_.empty()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, + "get_proof", std::move(P), td::Timestamp::in(3.0), std::move(q), + FullNode::max_proof_size() + FullNode::max_block_size() + 128, rldp_); + } else { + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_prepare", + create_serialize_tl_object_suffix(std::move(q)), + td::Timestamp::in(1.0), std::move(P)); + } +} + +void DownloadBlockNew::got_data(td::BufferSlice data) { + auto F = fetch_tl_object(std::move(data), true); + + if (F.is_error()) { + abort_query(F.move_as_error_prefix("received invalid answer: ")); + return; + } + + auto f = F.move_as_ok(); + + ton_api::downcast_call( + *f.get(), + td::overloaded( + [&](ton_api::tonNode_dataFullEmpty &x) { + abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block")); + }, + [&](ton_api::tonNode_dataFull &x) { + if (!allow_partial_proof_ && x.is_link_) { + abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block")); + return; + } + auto id = create_block_id(x.id_); + if (block_id_.is_valid() && id != block_id_) { + abort_query(td::Status::Error(ErrorCode::notready, "received data for wrong block")); + return; + } + block_.id = id; + block_.data = std::move(x.block_); + if (td::sha256_bits256(block_.data.as_slice()) != id.file_hash) { + abort_query(td::Status::Error(ErrorCode::notready, "received data with bad hash")); + return; + } + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, + R.move_as_error_prefix("received bad proof: ")); + } else { + td::actor::send_closure(SelfId, &DownloadBlockNew::checked_block_proof); + } + }); + if (block_id_.is_valid()) { + if (x.is_link_) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof_link, + block_id_, std::move(x.proof_), std::move(P)); + } else { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof, block_id_, + std::move(x.proof_), std::move(P)); + } + } else { + CHECK(!x.is_link_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_is_next_proof, + prev_id_, id, std::move(x.proof_), std::move(P)); + } + })); +} + +void DownloadBlockNew::got_data_from_db(td::BufferSlice data) { + block_.data = std::move(data); + finish_query(); +} + +void DownloadBlockNew::checked_block_proof() { + finish_query(); +} + +} // namespace fullnode + +} // namespace validator + +} // namespace ton diff --git a/validator/net/download-block-new.hpp b/validator/net/download-block-new.hpp new file mode 100644 index 00000000..8a5f91a2 --- /dev/null +++ b/validator/net/download-block-new.hpp @@ -0,0 +1,89 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#pragma once + +#include "overlay/overlays.h" +#include "ton/ton-types.h" +#include "validator/validator.h" +#include "rldp/rldp.h" +#include "adnl/adnl-ext-client.h" + +namespace ton { + +namespace validator { + +namespace fullnode { + +class DownloadBlockNew : public td::actor::Actor { + public: + DownloadBlockNew(BlockIdExt block_id, adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, + adnl::AdnlNodeIdShort download_from, td::uint32 priority, td::Timestamp timeout, + td::actor::ActorId validator_manager, td::actor::ActorId rldp, + td::actor::ActorId overlays, td::actor::ActorId adnl, + td::actor::ActorId client, td::Promise promise); + DownloadBlockNew(adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, BlockIdExt prev_id, + adnl::AdnlNodeIdShort download_from, td::uint32 priority, td::Timestamp timeout, + td::actor::ActorId validator_manager, td::actor::ActorId rldp, + td::actor::ActorId overlays, td::actor::ActorId adnl, + td::actor::ActorId client, td::Promise promise); + + void abort_query(td::Status reason); + void alarm() override; + void finish_query(); + + void start_up() override; + void got_block_handle(BlockHandle handle); + void got_download_token(std::unique_ptr token); + void got_node_to_download(adnl::AdnlNodeIdShort node); + void got_data(td::BufferSlice data); + void got_data_from_db(td::BufferSlice data); + void checked_block_proof(); + + private: + BlockIdExt block_id_; + adnl::AdnlNodeIdShort local_id_; + overlay::OverlayIdShort overlay_id_; + BlockIdExt prev_id_; + + adnl::AdnlNodeIdShort download_from_ = adnl::AdnlNodeIdShort::zero(); + + td::uint32 priority_; + + td::Timestamp timeout_; + td::actor::ActorId validator_manager_; + td::actor::ActorId rldp_; + td::actor::ActorId overlays_; + td::actor::ActorId adnl_; + td::actor::ActorId client_; + td::Promise promise_; + + BlockHandle handle_; + ReceivedBlock block_; + bool skip_proof_ = false; + + bool allow_partial_proof_ = false; + + std::unique_ptr token_; +}; + +} // namespace fullnode + +} // namespace validator + +} // namespace ton