diff --git a/Changelog.md b/Changelog.md index c8a7ea25..15e2450d 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,17 @@ +## 2024.12 Update + +1. FunC 0.4.6: Fix in try/catch handling, fixing pure flag for functions stored in variables +2. Merging parts of Accelerator: support of specific shard monitoring, archive/liteserver slice format, support for partial liteservers, proxy liteserver, on-demand neighbour queue loading +3. Fix of asynchronous cell loading +4. Various improvements: caching certificates checks, better block overloading detection, `_malloc` in emulator +5. Introduction of telemetry in overlays +6. Use non-null local-id for tonlib-LS interaction - mitigates MitM attack. +7. Adding `SECP256K1_XONLY_PUBKEY_TWEAK_ADD`, `SETCONTCTRMANY` instructions to TVM (activated by `Config8.version >= 9`) +8. Private keys export via validator-engine-console - required for better backups +9. Fix proof checking in tonlib, `hash` in `raw.Message` in tonlib_api + +Besides the work of the core team, this update is based on the efforts of OtterSec and LayerZero (FunC), tg:@throwunless (FunC), Aviv Frenkel and Dima Kogan from Fordefi (LS MitM), @hacker-volodya (Tonlib), OKX team (async cell loading), @krigga (emulator) + ## 2024.10 Update 1. Parallel write to celldb: substantial improvement of sync and GC speed, especially with slow disks. diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 31a60b56..72bffae5 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -272,6 +272,10 @@ class HardforkCreator : public td::actor::Actor { void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override { } + void download_out_msg_queue_proof( + ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise>> promise) override { + } void new_key_block(ton::validator::BlockHandle handle) override { } diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index 6b9ca8bf..98546a2d 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -660,6 +660,12 @@ bool EnqueuedMsgDescr::check_key(td::ConstBitPtr key) const { hash_ == key + 96; } +bool ImportedMsgQueueLimits::deserialize(vm::CellSlice& cs) { + return cs.fetch_ulong(8) == 0xd3 // imported_msg_queue_limits#d3 + && cs.fetch_uint_to(32, max_bytes) // max_bytes:# + && cs.fetch_uint_to(32, max_msgs); // max_msgs:# +} + bool ParamLimits::deserialize(vm::CellSlice& cs) { return cs.fetch_ulong(8) == 0xc3 // param_limits#c3 && cs.fetch_uint_to(32, limits_[0]) // underload:uint32 diff --git a/crypto/block/block.h b/crypto/block/block.h index 0247d79c..56e6dd38 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -216,6 +216,16 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC return proc_coll.print(os); } +struct ImportedMsgQueueLimits { + // Default values + td::uint32 max_bytes = 1 << 16; + td::uint32 max_msgs = 30; + bool deserialize(vm::CellSlice& cs); + ImportedMsgQueueLimits operator*(td::uint32 x) const { + return {max_bytes * x, max_msgs * x}; + } +}; + struct ParamLimits { enum { limits_cnt = 4 }; enum { cl_underload = 0, cl_normal = 1, cl_soft = 2, cl_medium = 3, cl_hard = 4 }; diff --git a/crypto/common/bitstring.h b/crypto/common/bitstring.h index 25776478..12333522 100644 --- a/crypto/common/bitstring.h +++ b/crypto/common/bitstring.h @@ -554,11 +554,7 @@ class BitArray { set_same(0); } void set_zero_s() { - volatile uint8* p = data(); - auto x = m; - while (x--) { - *p++ = 0; - } + as_slice().fill_zero_secure(); } void set_ones() { set_same(1); diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index b69cd8c0..d4deae4a 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -111,14 +111,16 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) override { + auto promise_ptr = std::make_shared>>(std::move(promise)); auto info = hash_table_.get_if_exists(hash); if (info && info->sync_with_db) { - TRY_RESULT_PROMISE(promise, loaded_cell, info->cell->load_cell()); - promise.set_result(loaded_cell.data_cell); + executor->execute_async([promise = std::move(promise_ptr), cell = info->cell]() mutable { + TRY_RESULT_PROMISE((*promise), loaded_cell, cell->load_cell()); + promise->set_result(loaded_cell.data_cell); + }); return; } SimpleExtCellCreator ext_cell_creator(cell_db_reader_); - auto promise_ptr = std::make_shared>>(std::move(promise)); executor->execute_async( [executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this, ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable { diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index 4b638300..61de96d3 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -47,8 +47,8 @@ endif() if (USE_EMSCRIPTEN) add_executable(emulator-emscripten ${EMULATOR_EMSCRIPTEN_SOURCE}) target_link_libraries(emulator-emscripten PUBLIC emulator) - target_link_options(emulator-emscripten PRIVATE -sEXPORTED_RUNTIME_METHODS=_malloc,free,UTF8ToString,stringToUTF8,allocate,ALLOC_NORMAL,lengthBytesUTF8) - target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator,_version) + target_link_options(emulator-emscripten PRIVATE -sEXPORTED_RUNTIME_METHODS=UTF8ToString,stringToUTF8,allocate,ALLOC_NORMAL,lengthBytesUTF8) + target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_malloc,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator,_version) target_link_options(emulator-emscripten PRIVATE -sEXPORT_NAME=EmulatorModule) target_link_options(emulator-emscripten PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0) target_link_options(emulator-emscripten PRIVATE -Oz) diff --git a/keyring/keyring.cpp b/keyring/keyring.cpp index 0f45879d..a31d173e 100644 --- a/keyring/keyring.cpp +++ b/keyring/keyring.cpp @@ -28,7 +28,7 @@ namespace ton { namespace keyring { KeyringImpl::PrivateKeyDescr::PrivateKeyDescr(PrivateKey private_key, bool is_temp) - : public_key(private_key.compute_public_key()), is_temp(is_temp) { + : public_key(private_key.compute_public_key()), private_key(private_key), is_temp(is_temp) { auto D = private_key.create_decryptor_async(); D.ensure(); decryptor_sign = D.move_as_ok(); @@ -190,6 +190,16 @@ void KeyringImpl::decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, } } +void KeyringImpl::export_all_private_keys(td::Promise> promise) { + std::vector keys; + for (auto& [_, descr] : map_) { + if (!descr->is_temp && descr->private_key.exportable()) { + keys.push_back(descr->private_key); + } + } + promise.set_value(std::move(keys)); +} + td::actor::ActorOwn Keyring::create(std::string db_root) { return td::actor::create_actor("keyring", db_root); } diff --git a/keyring/keyring.h b/keyring/keyring.h index 044d8d29..3b9064a7 100644 --- a/keyring/keyring.h +++ b/keyring/keyring.h @@ -44,6 +44,8 @@ class Keyring : public td::actor::Actor { virtual void decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, td::Promise promise) = 0; + virtual void export_all_private_keys(td::Promise> promise) = 0; + static td::actor::ActorOwn create(std::string db_root); }; diff --git a/keyring/keyring.hpp b/keyring/keyring.hpp index ec658305..eca9073a 100644 --- a/keyring/keyring.hpp +++ b/keyring/keyring.hpp @@ -33,6 +33,7 @@ class KeyringImpl : public Keyring { td::actor::ActorOwn decryptor_sign; td::actor::ActorOwn decryptor_decrypt; PublicKey public_key; + PrivateKey private_key; bool is_temp; PrivateKeyDescr(PrivateKey private_key, bool is_temp); }; @@ -56,6 +57,8 @@ class KeyringImpl : public Keyring { void decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, td::Promise promise) override; + void export_all_private_keys(td::Promise> promise) override; + KeyringImpl(std::string db_root) : db_root_(db_root) { } diff --git a/recent_changelog.md b/recent_changelog.md index fb7433e5..cc877c2c 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,17 +1,13 @@ -## 2024.10 Update - -1. Parallel write to celldb: substantial improvement of sync and GC speed, especially with slow disks. -2. Decreased network traffic: only first block candidate is sent optimistically. -3. Improved channel creation and dht lookups, introduction of semi-private overlays -4. New LS dispatch queue related methods and improvement security -5. Fixing recursion in TVM continuations -6. Improved stats for actors, validator sessions, perf counters, overlays, adnl, rocksdb -7. Migration to C++20 -8. Improved block size estimates: account for depth in various structures -9. Fix bug with `<<` optimization in FunC -10. Minor changes of TVM which will be activated by `Config8.version >= 9` -11. Multiple minor improvements - -Besides the work of the core team, this update is based on the efforts of @krigga (emulator), Arayz @ TonBit (LS security, TVM recursion), @ret2happy (UB in BLST). +## 2024.12 Update +1. FunC 0.4.6: Fix in try/catch handling, fixing pure flag for functions stored in variables +2. Merging parts of Accelerator: support of specific shard monitoring, archive/liteserver slice format, support for partial liteservers, proxy liteserver, on-demand neighbour queue loading +3. Fix of asynchronous cell loading +4. Various improvements: caching certificates checks, better block overloading detection, `_malloc` in emulator +5. Introduction of telemetry in overlays +6. Use non-null local-id for tonlib-LS interaction - mitigates MitM attack. +7. Adding `SECP256K1_XONLY_PUBKEY_TWEAK_ADD`, `SETCONTCTRMANY` instructions to TVM (activated by `Config8.version >= 9`) +8. Private keys export via validator-engine-console - required for better backups +9. Fix proof checking in tonlib, `hash` in `raw.Message` in tonlib_api +Besides the work of the core team, this update is based on the efforts of OtterSec and LayerZero (FunC), tg:@throwunless (FunC), Aviv Frenkel and Dima Kogan from Fordefi (LS MitM), @hacker-volodya (Tonlib), OKX team (async cell loading), @krigga (emulator) diff --git a/tdutils/td/utils/StringBuilder.h b/tdutils/td/utils/StringBuilder.h index 99e9d517..685416fe 100644 --- a/tdutils/td/utils/StringBuilder.h +++ b/tdutils/td/utils/StringBuilder.h @@ -149,4 +149,19 @@ std::enable_if_t::value, string> to_string(const T &x) { return sb.as_cslice().str(); } +template +struct LambdaPrintHelper { + SB& sb; +}; +template +SB& operator<<(const LambdaPrintHelper& helper, F&& f) { + f(helper.sb); + return helper.sb; +} +struct LambdaPrint {}; + +inline LambdaPrintHelper operator<<(td::StringBuilder& sb, const LambdaPrint&) { + return LambdaPrintHelper{sb}; +} + } // namespace td diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index d00fba15..5c9a0621 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -74,6 +74,7 @@ #define LOG(level) LOG_IMPL(level, level, true, ::td::Slice()) #define LOG_IF(level, condition) LOG_IMPL(level, level, condition, #condition) +#define FLOG(level) LOG_IMPL(level, level, true, ::td::Slice()) << td::LambdaPrint{} << [&](auto &sb) #define VLOG(level) LOG_IMPL(DEBUG, level, true, TD_DEFINE_STR(level)) #define VLOG_IF(level, condition) LOG_IMPL(DEBUG, level, condition, TD_DEFINE_STR(level) " " #condition) @@ -95,13 +96,13 @@ inline bool no_return_func() { #define DUMMY_LOG_CHECK(condition) LOG_IF(NEVER, !(condition)) #ifdef TD_DEBUG - #if TD_MSVC +#if TD_MSVC #define LOG_CHECK(condition) \ __analysis_assume(!!(condition)); \ LOG_IMPL(FATAL, FATAL, !(condition), #condition) - #else +#else #define LOG_CHECK(condition) LOG_IMPL(FATAL, FATAL, !(condition) && no_return_func(), #condition) - #endif +#endif #else #define LOG_CHECK DUMMY_LOG_CHECK #endif @@ -263,6 +264,9 @@ class Logger { sb_ << other; return *this; } + LambdaPrintHelper operator<<(const LambdaPrint &) { + return LambdaPrintHelper{*this}; + } MutableCSlice as_cslice() { return sb_.as_cslice(); diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 11824851..0fde1e68 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -373,6 +373,10 @@ class TestNode : public td::actor::Actor { void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override { } + void download_out_msg_queue_proof( + ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise>> promise) override { + } void new_key_block(ton::validator::BlockHandle handle) override { } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 16f48345..cfc9f3a1 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -454,6 +454,10 @@ tonNode.success = tonNode.Success; tonNode.archiveNotFound = tonNode.ArchiveInfo; tonNode.archiveInfo id:long = tonNode.ArchiveInfo; +tonNode.importedMsgQueueLimits max_bytes:int max_msgs:int = ImportedMsgQueueLimits; +tonNode.outMsgQueueProof queue_proofs:bytes block_state_proofs:bytes msg_counts:(vector int) = tonNode.OutMsgQueueProof; +tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; + tonNode.forgetPeer = tonNode.ForgetPeer; ---functions--- @@ -483,6 +487,8 @@ tonNode.downloadKeyBlockProofLink block:tonNode.blockIdExt = tonNode.Data; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; tonNode.getShardArchiveInfo masterchain_seqno:int shard_prefix:tonNode.shardId = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; +tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt) + limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; @@ -725,6 +731,8 @@ engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByN engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize; +engine.validator.exportedPrivateKeys encrypted_data:bytes = engine.validator.ExportedPrivateKeys; + ---functions--- @@ -755,6 +763,7 @@ engine.validator.delListeningPort ip:int port:int categories:(vector int) priori engine.validator.delProxy out_ip:int out_port:int categories:(vector int) priority_categories:(vector int) = engine.validator.Success; engine.validator.sign key_hash:int256 data:bytes = engine.validator.Signature; +engine.validator.exportAllPrivateKeys encryption_key:PublicKey = engine.validator.ExportedPrivateKeys; engine.validator.getStats = engine.validator.Stats; engine.validator.getConfig = engine.validator.JsonConfig; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 1689260b..96ecb775 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/ton/ton-types.h b/ton/ton-types.h index cd970081..2447a8c5 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -120,6 +120,26 @@ struct ShardIdFull { char buffer[64]; return std::string{buffer, (unsigned)snprintf(buffer, 63, "(%d,%016llx)", workchain, (unsigned long long)shard)}; } + static td::Result parse(td::Slice s) { + // Formats: (0,2000000000000000) (0:2000000000000000) 0,2000000000000000 0:2000000000000000 + if (s.empty()) { + return td::Status::Error("empty string"); + } + if (s[0] == '(' && s.back() == ')') { + s = s.substr(1, s.size() - 2); + } + auto sep = s.find(':'); + if (sep == td::Slice::npos) { + sep = s.find(','); + } + if (sep == td::Slice::npos || s.size() - sep - 1 != 16) { + return td::Status::Error(PSTRING() << "invalid shard " << s); + } + ShardIdFull shard; + TRY_RESULT_ASSIGN(shard.workchain, td::to_integer_safe(s.substr(0, sep))); + TRY_RESULT_ASSIGN(shard.shard, td::hex_to_integer_safe(s.substr(sep + 1))); + return shard; + } }; struct AccountIdPrefixFull { diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index b9ff4899..507512d0 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -5673,6 +5673,26 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& reques td::Status check_lookup_block_proof(lite_api_ptr& result, int mode, ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, td::uint32 utime) { try { ton::BlockIdExt cur_id = ton::create_block_id(result->mc_block_id_); + if (!cur_id.is_masterchain_ext()) { + return td::Status::Error("invalid response: mc block id is not from masterchain"); + } + if (client_mc_blkid != cur_id) { + auto state = block::check_extract_state_proof(client_mc_blkid, result->client_mc_state_proof_.as_slice(), + result->mc_block_proof_.as_slice()); + if (state.is_error()) { + LOG(WARNING) << "cannot check state proof: " << state.move_as_error().to_string(); + return state.move_as_error(); + } + auto state_root = state.move_as_ok(); + auto prev_blocks_dict = block::get_prev_blocks_dict(state_root); + if (!prev_blocks_dict) { + return td::Status::Error("cannot extract prev blocks dict from state"); + } + + if (!block::check_old_mc_block_id(*prev_blocks_dict, cur_id)) { + return td::Status::Error("couldn't check old mc block id"); + } + } try { for (auto& link : result->shard_links_) { ton::BlockIdExt prev_id = create_block_id(link->id_); @@ -5686,23 +5706,6 @@ td::Status check_lookup_block_proof(lite_api_ptrclient_mc_state_proof_.as_slice(), - result->mc_block_proof_.as_slice()); - if (state.is_error()) { - LOG(WARNING) << "cannot check state proof: " << state.move_as_error().to_string(); - return state.move_as_error(); - } - auto state_root = state.move_as_ok(); - auto prev_blocks_dict = block::get_prev_blocks_dict(state_root); - if (!prev_blocks_dict) { - return td::Status::Error("cannot extract prev blocks dict from state"); - } - - if (!block::check_old_mc_block_id(*prev_blocks_dict, cur_id)) { - return td::Status::Error("couldn't check old mc block id"); - } - } block::gen::Block::Record blk; block::gen::BlockExtra::Record extra; block::gen::McBlockExtra::Record mc_extra; diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 926b7988..d1110019 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -35,6 +35,8 @@ #include "ton/ton-tl.hpp" #include "td/utils/JsonBuilder.h" #include "auto/tl/ton_api_json.h" +#include "keys/encryptor.h" +#include "td/utils/port/path.h" #include "tl/tl_json.h" #include @@ -283,6 +285,66 @@ td::Status SignFileQuery::receive(td::BufferSlice data) { return td::Status::OK(); } +td::Status ExportAllPrivateKeysQuery::run() { + TRY_RESULT_ASSIGN(directory_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + client_pk_ = ton::privkeys::Ed25519::random(); + return td::Status::OK(); +} + +td::Status ExportAllPrivateKeysQuery::send() { + auto b = ton::create_serialize_tl_object( + client_pk_.compute_public_key().tl()); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ExportAllPrivateKeysQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + // Private keys are encrypted using client-provided public key to avoid storing them in + // non-secure buffers (not td::SecureString) + TRY_RESULT_PREFIX(decryptor, client_pk_.create_decryptor(), "cannot create decryptor: "); + TRY_RESULT_PREFIX(keys_data, decryptor->decrypt(f->encrypted_data_.as_slice()), "cannot decrypt data: "); + SCOPE_EXIT { + keys_data.as_slice().fill_zero_secure(); + }; + td::Slice slice = keys_data.as_slice(); + if (slice.size() < 32) { + return td::Status::Error("data is too small"); + } + slice.remove_suffix(32); + std::vector private_keys; + while (!slice.empty()) { + if (slice.size() < 4) { + return td::Status::Error("unexpected end of data"); + } + td::uint32 size; + td::MutableSlice{reinterpret_cast(&size), 4}.copy_from(slice.substr(0, 4)); + if (size > slice.size()) { + return td::Status::Error("unexpected end of data"); + } + slice.remove_prefix(4); + TRY_RESULT_PREFIX(private_key, ton::PrivateKey::import(slice.substr(0, size)), "cannot parse private key: "); + if (!private_key.exportable()) { + return td::Status::Error("private key is not exportable"); + } + private_keys.push_back(std::move(private_key)); + slice.remove_prefix(size); + } + + TRY_STATUS_PREFIX(td::mkpath(directory_ + "/"), "cannot create directory " + directory_ + ": "); + td::TerminalIO::out() << "exported " << private_keys.size() << " private keys" << "\n"; + for (const ton::PrivateKey &private_key : private_keys) { + std::string hash_hex = private_key.compute_short_id().bits256_value().to_hex(); + TRY_STATUS_PREFIX(td::write_file(directory_ + "/" + hash_hex, private_key.export_as_slice()), + "failed to write file: "); + td::TerminalIO::out() << "pubkey_hash " << hash_hex << "\n"; + } + td::TerminalIO::out() << "written all files to " << directory_ << "\n"; + return td::Status::OK(); +} + td::Status AddAdnlAddrQuery::run() { TRY_RESULT_ASSIGN(key_hash_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(category_, tokenizer_.get_token()); @@ -979,8 +1041,7 @@ td::Status ImportCertificateQuery::receive(td::BufferSlice data) { td::Status SignShardOverlayCertificateQuery::run() { - TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); @@ -990,8 +1051,9 @@ td::Status SignShardOverlayCertificateQuery::run() { } td::Status SignShardOverlayCertificateQuery::send() { - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); + auto b = ton::create_serialize_tl_object( + shard_.workchain, shard_.shard, ton::create_tl_object(key_.tl()), + expire_at_, max_size_); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1009,8 +1071,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { } td::Status ImportShardOverlayCertificateQuery::run() { - TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); @@ -1021,8 +1082,9 @@ td::Status ImportShardOverlayCertificateQuery::send() { TRY_RESULT(data, td::read_file(in_file_)); TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), "incorrect certificate"); - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); + auto b = ton::create_serialize_tl_object( + shard_.workchain, shard_.shard, ton::create_tl_object(key_.tl()), + std::move(cert)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1111,14 +1173,12 @@ td::Status GetPerfTimerStatsJsonQuery::receive(td::BufferSlice data) { } td::Status GetShardOutQueueSizeQuery::run() { - TRY_RESULT_ASSIGN(block_id_.workchain, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(block_id_.shard, tokenizer_.get_token()); + TRY_RESULT(shard, tokenizer_.get_token()); + block_id_.workchain = shard.workchain; + block_id_.shard = shard.shard; TRY_RESULT_ASSIGN(block_id_.seqno, tokenizer_.get_token()); if (!tokenizer_.endl()) { - ton::ShardIdFull dest; - TRY_RESULT_ASSIGN(dest.workchain, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(dest.shard, tokenizer_.get_token()); - dest_ = dest; + TRY_RESULT_ASSIGN(dest_, tokenizer_.get_token()); } TRY_STATUS(tokenizer_.check_endl()); return td::Status::OK(); @@ -1126,8 +1186,7 @@ td::Status GetShardOutQueueSizeQuery::run() { td::Status GetShardOutQueueSizeQuery::send() { auto b = ton::create_serialize_tl_object( - dest_ ? 1 : 0, ton::create_tl_block_id_simple(block_id_), dest_ ? dest_.value().workchain : 0, - dest_ ? dest_.value().shard : 0); + dest_.is_valid() ? 1 : 0, ton::create_tl_block_id_simple(block_id_), dest_.workchain, dest_.shard); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1501,14 +1560,13 @@ td::Status GetAdnlStatsQuery::receive(td::BufferSlice data) { } td::Status AddShardQuery::run() { - TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); return td::Status::OK(); } td::Status AddShardQuery::send() { - auto b = ton::create_serialize_tl_object( - ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_))); + auto b = ton::create_serialize_tl_object(ton::create_tl_shard_id(shard_)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1521,14 +1579,13 @@ td::Status AddShardQuery::receive(td::BufferSlice data) { } td::Status DelShardQuery::run() { - TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); return td::Status::OK(); } td::Status DelShardQuery::send() { - auto b = ton::create_serialize_tl_object( - ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_))); + auto b = ton::create_serialize_tl_object(ton::create_tl_shard_id(shard_)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 4ea172a3..817d70c9 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -36,6 +36,7 @@ #include "ton/ton-types.h" #include "keys/keys.hpp" +#include "td/utils/base64.h" class ValidatorEngineConsole; @@ -95,27 +96,25 @@ inline td::Result Tokenizer::get_token() { } template <> -inline td::Result Tokenizer::get_token() { - TRY_RESULT(S, get_raw_token()); - TRY_RESULT(F, td::hex_decode(S)); - if (F.size() == 32) { - return ton::PublicKeyHash{td::Slice{F}}; +inline td::Result Tokenizer::get_token() { + TRY_RESULT(word, get_raw_token()); + std::string data; + if (word.size() == 64) { + TRY_RESULT_ASSIGN(data, td::hex_decode(word)); + } else if (word.size() == 44) { + TRY_RESULT_ASSIGN(data, td::base64_decode(word)); } else { return td::Status::Error("cannot parse keyhash: bad length"); } + td::Bits256 v; + v.as_slice().copy_from(data); + return v; } template <> -inline td::Result Tokenizer::get_token() { - TRY_RESULT(S, get_raw_token()); - TRY_RESULT(F, td::hex_decode(S)); - if (F.size() == 32) { - td::Bits256 v; - v.as_slice().copy_from(F); - return v; - } else { - return td::Status::Error("cannot parse keyhash: bad length"); - } +inline td::Result Tokenizer::get_token() { + TRY_RESULT(x, get_token()); + return ton::PublicKeyHash{x}; } template <> @@ -146,6 +145,18 @@ inline td::Result> Tokenizer::get_token_vector() { } } +template <> +inline td::Result Tokenizer::get_token() { + TRY_RESULT(word, get_raw_token()); + auto r_wc = td::to_integer_safe(word); + if (r_wc.is_ok()) { + TRY_RESULT_ASSIGN(word, get_raw_token()); + TRY_RESULT(shard, td::to_integer_safe(word)); + return ton::ShardIdFull{r_wc.move_as_ok(), shard}; + } + return ton::ShardIdFull::parse(word); +} + class QueryRunner { public: virtual ~QueryRunner() = default; @@ -222,10 +233,10 @@ class GetTimeQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "gettime"; + return "get-time"; } static std::string get_help() { - return "gettime\tshows current server unixtime"; + return "get-time\tshows current server unixtime"; } std::string name() const override { return get_name(); @@ -287,10 +298,10 @@ class NewKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "newkey"; + return "new-key"; } static std::string get_help() { - return "newkey\tgenerates new key pair on server"; + return "new-key\tgenerates new key pair on server"; } std::string name() const override { return get_name(); @@ -308,10 +319,10 @@ class ImportPrivateKeyFileQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "importf"; + return "import-f"; } static std::string get_help() { - return "importf \timport private key"; + return "import-f \timport private key"; } std::string name() const override { return get_name(); @@ -330,10 +341,10 @@ class ExportPublicKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "exportpub"; + return "export-pub"; } static std::string get_help() { - return "exportpub \texports public key by key hash"; + return "export-pub \texports public key by key hash"; } std::string name() const override { return get_name(); @@ -352,10 +363,10 @@ class ExportPublicKeyFileQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "exportpubf"; + return "export-pubf"; } static std::string get_help() { - return "exportpubf \texports public key by key hash"; + return "export-pub-f \texports public key by key hash"; } std::string name() const override { return get_name(); @@ -398,10 +409,10 @@ class SignFileQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "signf"; + return "sign-f"; } static std::string get_help() { - return "signf \tsigns bytestring with privkey"; + return "sign-f \tsigns bytestring with privkey"; } std::string name() const override { return get_name(); @@ -413,6 +424,30 @@ class SignFileQuery : public Query { std::string out_file_; }; +class ExportAllPrivateKeysQuery : public Query { + public: + ExportAllPrivateKeysQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice R) override; + static std::string get_name() { + return "export-all-private-keys"; + } + static std::string get_help() { + return "export-all-private-keys \texports all private keys from validator engine and stores them to " + ""; + } + std::string name() const override { + return get_name(); + } + + private: + std::string directory_; + ton::PrivateKey client_pk_; +}; + class AddAdnlAddrQuery : public Query { public: AddAdnlAddrQuery(td::actor::ActorId console, Tokenizer tokenizer) @@ -422,10 +457,10 @@ class AddAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addadnl"; + return "add-adnl"; } static std::string get_help() { - return "addadnl \tuse key as ADNL addr"; + return "add-adnl \tuse key as ADNL addr"; } std::string name() const override { return get_name(); @@ -445,10 +480,10 @@ class AddDhtIdQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "adddht"; + return "add-dht"; } static std::string get_help() { - return "adddht \tcreate DHT node with specified ADNL addr"; + return "add-dht \tcreate DHT node with specified ADNL addr"; } std::string name() const override { return get_name(); @@ -467,10 +502,10 @@ class AddValidatorPermanentKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addpermkey"; + return "add-perm-key"; } static std::string get_help() { - return "addpermkey \tadd validator permanent key"; + return "add-perm-key \tadd validator permanent key"; } std::string name() const override { return get_name(); @@ -491,10 +526,10 @@ class AddValidatorTempKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addtempkey"; + return "add-temp-key"; } static std::string get_help() { - return "addtempkey \tadd validator temp key"; + return "add-temp-key \tadd validator temp key"; } std::string name() const override { return get_name(); @@ -515,10 +550,10 @@ class AddValidatorAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addvalidatoraddr"; + return "add-validator-addr"; } static std::string get_help() { - return "addvalidatoraddr \tadd validator ADNL addr"; + return "add-validator-addr \tadd validator ADNL addr"; } std::string name() const override { return get_name(); @@ -539,10 +574,10 @@ class ChangeFullNodeAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "changefullnodeaddr"; + return "change-full-node-addr"; } static std::string get_help() { - return "changefullnodeaddr \tchanges fullnode ADNL address"; + return "change-full-node-addr \tchanges fullnode ADNL address"; } std::string name() const override { return get_name(); @@ -561,10 +596,10 @@ class AddLiteServerQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addliteserver"; + return "add-liteserver"; } static std::string get_help() { - return "addliteserver \tadd liteserver"; + return "add-liteserver \tadd liteserver"; } std::string name() const override { return get_name(); @@ -584,10 +619,10 @@ class DelAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "deladnl"; + return "del-adnl"; } static std::string get_help() { - return "deladnl \tdel unused ADNL addr"; + return "del-adnl \tdel unused ADNL addr"; } std::string name() const override { return get_name(); @@ -606,10 +641,10 @@ class DelDhtIdQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "deldht"; + return "del-dht"; } static std::string get_help() { - return "deldht \tdel unused DHT node"; + return "del-dht \tdel unused DHT node"; } std::string name() const override { return get_name(); @@ -628,10 +663,10 @@ class DelValidatorPermanentKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "delpermkey"; + return "del-perm-key"; } static std::string get_help() { - return "delpermkey \tforce del unused validator permanent key"; + return "del-perm-key \tforce del unused validator permanent key"; } std::string name() const override { return get_name(); @@ -650,10 +685,10 @@ class DelValidatorTempKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "deltempkey"; + return "del-temp-key"; } static std::string get_help() { - return "deltempkey \tforce del unused validator temp key"; + return "del-temp-key \tforce del unused validator temp key"; } std::string name() const override { return get_name(); @@ -673,10 +708,10 @@ class DelValidatorAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "delvalidatoraddr"; + return "del-validator-addr"; } static std::string get_help() { - return "delvalidatoraddr \tforce del unused validator ADNL addr"; + return "del-validator-addr \tforce del unused validator ADNL addr"; } std::string name() const override { return get_name(); @@ -696,10 +731,10 @@ class GetConfigQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getconfig"; + return "get-config"; } static std::string get_help() { - return "getconfig\tdownloads current config"; + return "get-config\tdownloads current config"; } std::string name() const override { return get_name(); @@ -717,10 +752,10 @@ class SetVerbosityQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "setverbosity"; + return "set-verbosity"; } static std::string get_help() { - return "setverbosity \tchanges verbosity level"; + return "set-verbosity \tchanges verbosity level"; } std::string name() const override { return get_name(); @@ -739,10 +774,10 @@ class GetStatsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getstats"; + return "get-stats"; } static std::string get_help() { - return "getstats\tprints stats"; + return "get-stats\tprints stats"; } std::string name() const override { return get_name(); @@ -783,10 +818,10 @@ class AddNetworkAddressQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addaddr"; + return "add-addr"; } static std::string get_help() { - return "addaddr {cats...} {priocats...}\tadds ip address to address list"; + return "add-addr {cats...} {priocats...}\tadds ip address to address list"; } std::string name() const override { return get_name(); @@ -807,10 +842,10 @@ class AddNetworkProxyAddressQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addproxyaddr"; + return "add-proxy-addr"; } static std::string get_help() { - return "addproxyaddr {cats...} {priocats...}\tadds ip address to address list"; + return "add-proxy-addr {cats...} {priocats...}\tadds ip address to address list"; } std::string name() const override { return get_name(); @@ -834,10 +869,10 @@ class CreateElectionBidQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "createelectionbid"; + return "create-election-bid"; } static std::string get_help() { - return "createelectionbid \tcreate election bid"; + return "create-election-bid \tcreate election bid"; } std::string name() const override { return get_name(); @@ -859,10 +894,10 @@ class CreateProposalVoteQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "createproposalvote"; + return "create-proposal-vote"; } static std::string get_help() { - return "createproposalvote \tcreate proposal vote"; + return "create-proposal-vote \tcreate proposal vote"; } std::string name() const override { return get_name(); @@ -882,10 +917,10 @@ class CreateComplaintVoteQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "createcomplaintvote"; + return "create-complaint-vote"; } static std::string get_help() { - return "createcomplaintvote \tcreate proposal vote"; + return "create-complaint-vote \tcreate proposal vote"; } std::string name() const override { return get_name(); @@ -906,10 +941,10 @@ class CheckDhtServersQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "checkdht"; + return "check-dht"; } static std::string get_help() { - return "checkdht \tchecks, which root DHT servers are accessible from this ADNL addr"; + return "check-dht \tchecks, which root DHT servers are accessible from this ADNL addr"; } std::string name() const override { return get_name(); @@ -928,10 +963,10 @@ class GetOverlaysStatsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getoverlaysstats"; + return "get-overlays-stats"; } static std::string get_help() { - return "getoverlaysstats\tgets stats for all overlays"; + return "get-overlays-stats\tgets stats for all overlays"; } std::string name() const override { return get_name(); @@ -947,10 +982,10 @@ class GetOverlaysStatsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getoverlaysstatsjson"; + return "get-overlays-stats-json"; } static std::string get_help() { - return "getoverlaysstatsjson \tgets stats for all overlays and writes to json file"; + return "get-overlays-stats-json \tgets stats for all overlays and writes to json file"; } std::string name() const override { return get_name(); @@ -969,10 +1004,11 @@ class SignCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "signcert"; + return "sign-cert"; } static std::string get_help() { - return "signcert \tsign overlay certificate by key"; + return "sign-cert \tsign overlay certificate by " + " key"; } std::string name() const override { return get_name(); @@ -1005,10 +1041,10 @@ class ImportCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "importcert"; + return "import-cert"; } static std::string get_help() { - return "importcert \timport overlay certificate for specific key"; + return "import-cert \timport overlay certificate for specific key"; } std::string name() const override { return get_name(); @@ -1030,10 +1066,11 @@ class SignShardOverlayCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "signshardoverlaycert"; + return "sign-shard-overlay-cert"; } static std::string get_help() { - return "signshardoverlaycert \tsign certificate for in currently active shard overlay"; + return "sign-shard-overlay-cert : \tsign certificate " + "for in currently active shard overlay"; } std::string name() const override { return get_name(); @@ -1041,8 +1078,7 @@ class SignShardOverlayCertificateQuery : public Query { private: - td::int32 wc_; - td::int64 shard_; + ton::ShardIdFull shard_; td::int32 expire_at_; ton::PublicKeyHash key_; td::uint32 max_size_; @@ -1059,10 +1095,11 @@ class ImportShardOverlayCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "importshardoverlaycert"; + return "import-shard-overlay-cert"; } static std::string get_help() { - return "importshardoverlaycert \timport certificate for in currently active shard overlay"; + return "import-shard-overlay-cert : \timport certificate for in " + "currently active shard overlay"; } std::string name() const override { return get_name(); @@ -1070,8 +1107,7 @@ class ImportShardOverlayCertificateQuery : public Query { private: - td::int32 wc_; - td::int64 shard_; + ton::ShardIdFull shard_; ton::PublicKeyHash key_; std::string in_file_; }; @@ -1085,10 +1121,10 @@ class GetActorStatsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getactorstats"; + return "get-actor-stats"; } static std::string get_help() { - return "getactorstats []\tget actor stats and print it either in stdout or in "; + return "get-actor-stats []\tget actor stats and print it either in stdout or in "; } std::string name() const override { return get_name(); @@ -1107,10 +1143,11 @@ class GetPerfTimerStatsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getperftimerstatsjson"; + return "get-perf-timer-stats-json"; } static std::string get_help() { - return "getperftimerstatsjson \tgets min, average and max event processing time for last 60, 300 and 3600 seconds and writes to json file"; + return "get-perf-timer-stats-json \tgets min, average and max event processing time for last 60, 300 and " + "3600 seconds and writes to json file"; } std::string name() const override { return get_name(); @@ -1129,10 +1166,10 @@ class GetShardOutQueueSizeQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getshardoutqueuesize"; + return "get-shard-out-queue-size"; } static std::string get_help() { - return "getshardoutqueuesize [ ]\treturns number of messages in the " + return "get-shard-out-queue-size : [:]\treturns number of messages in the " "queue of the given shard. Destination shard is optional."; } std::string name() const override { @@ -1141,7 +1178,7 @@ class GetShardOutQueueSizeQuery : public Query { private: ton::BlockId block_id_; - td::optional dest_; + ton::ShardIdFull dest_ = ton::ShardIdFull{ton::workchainInvalid}; }; class SetExtMessagesBroadcastDisabledQuery : public Query { @@ -1153,11 +1190,11 @@ class SetExtMessagesBroadcastDisabledQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "setextmessagesbroadcastdisabled"; + return "set-ext-messages-broadcast-disabled"; } static std::string get_help() { - return "setextmessagesbroadcastdisabled \tdisable broadcasting and rebroadcasting ext messages; value is 0 " - "or 1."; + return "set-ext-messages-broadcast-disabled \tdisable broadcasting and rebroadcasting ext messages; value " + "is 0 or 1."; } std::string name() const override { return get_name(); @@ -1176,10 +1213,10 @@ class AddCustomOverlayQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addcustomoverlay"; + return "add-custom-overlay"; } static std::string get_help() { - return "addcustomoverlay \tadd custom overlay with config from file "; + return "add-custom-overlay \tadd custom overlay with config from file "; } std::string name() const override { return get_name(); @@ -1198,10 +1235,10 @@ class DelCustomOverlayQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "delcustomoverlay"; + return "del-custom-overlay"; } static std::string get_help() { - return "delcustomoverlay \tdelete custom overlay with name "; + return "del-custom-overlay \tdelete custom overlay with name "; } std::string name() const override { return get_name(); @@ -1220,10 +1257,10 @@ class ShowCustomOverlaysQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "showcustomoverlays"; + return "show-custom-overlays"; } static std::string get_help() { - return "showcustomoverlays\tshow all custom overlays"; + return "show-custom-overlays\tshow all custom overlays"; } std::string name() const override { return get_name(); @@ -1239,10 +1276,10 @@ class SetStateSerializerEnabledQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "setstateserializerenabled"; + return "set-state-serializer-enabled"; } static std::string get_help() { - return "setstateserializerenabled \tdisable or enable persistent state serializer; value is 0 or 1"; + return "set-state-serializer-enabled \tdisable or enable persistent state serializer; value is 0 or 1"; } std::string name() const override { return get_name(); @@ -1261,10 +1298,10 @@ class SetCollatorOptionsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "setcollatoroptionsjson"; + return "set-collator-options-json"; } static std::string get_help() { - return "setcollatoroptionsjson \tset collator options from file "; + return "set-collator-options-json \tset collator options from file "; } std::string name() const override { return get_name(); @@ -1283,10 +1320,10 @@ class ResetCollatorOptionsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "resetcollatoroptions"; + return "reset-collator-options"; } static std::string get_help() { - return "resetcollatoroptions\tset collator options to default values"; + return "reset-collator-options\tset collator options to default values"; } std::string name() const override { return get_name(); @@ -1302,10 +1339,10 @@ class GetCollatorOptionsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getcollatoroptionsjson"; + return "get-collator-options-json"; } static std::string get_help() { - return "getcollatoroptionsjson \tsave current collator options to file "; + return "get-collator-options-json \tsave current collator options to file "; } std::string name() const override { return get_name(); @@ -1324,11 +1361,11 @@ class GetAdnlStatsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getadnlstatsjson"; + return "get-adnl-stats-json"; } static std::string get_help() { - return "getadnlstatsjson [all]\tsave adnl stats to . all - returns all peers (default - only " - "peers with traffic in the last 10 minutes)"; + return "get-adnl-stats-json [all]\tsave adnl stats to . all - returns all peers (default - " + "only peers with traffic in the last 10 minutes)"; } std::string name() const override { return get_name(); @@ -1348,11 +1385,11 @@ class GetAdnlStatsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getadnlstats"; + return "get-adnl-stats"; } static std::string get_help() { - return "getadnlstats [all]\tdisplay adnl stats. all - returns all peers (default - only peers with traffic in the " - "last 10 minutes)"; + return "get-adnl-stats [all]\tdisplay adnl stats. all - returns all peers (default - only peers with traffic in " + "the last 10 minutes)"; } std::string name() const override { return get_name(); @@ -1372,18 +1409,17 @@ class AddShardQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addshard"; + return "add-shard"; } static std::string get_help() { - return "addshard \tstart monitoring shard"; + return "add-shard :\tstart monitoring shard"; } std::string name() const override { return get_name(); } private: - td::int32 wc_; - td::int64 shard_; + ton::ShardIdFull shard_; }; class DelShardQuery : public Query { @@ -1395,16 +1431,15 @@ class DelShardQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "delshard"; + return "del-shard"; } static std::string get_help() { - return "delshard \tstop monitoring shard"; + return "del-shard :\tstop monitoring shard"; } std::string name() const override { return get_name(); } private: - td::int32 wc_; - td::int64 shard_; + ton::ShardIdFull shard_; }; \ No newline at end of file diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 1f348e28..234cd6a5 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -112,6 +112,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); @@ -205,9 +206,8 @@ void ValidatorEngineConsole::show_help(std::string command, td::Promisehelp() << "\n"; } } else { - auto it = query_runners_.find(command); - if (it != query_runners_.end()) { - td::TerminalIO::out() << it->second->help() << "\n"; + if (auto query = get_query(command)) { + td::TerminalIO::out() << query->help() << "\n"; } else { td::TerminalIO::out() << "unknown command '" << command << "'\n"; } @@ -231,10 +231,9 @@ void ValidatorEngineConsole::parse_line(td::BufferSlice data) { } auto name = tokenizer.get_token().move_as_ok(); - auto it = query_runners_.find(name); - if (it != query_runners_.end()) { + if (auto query = get_query(name)) { running_queries_++; - it->second->run(actor_id(this), std::move(tokenizer)); + query->run(actor_id(this), std::move(tokenizer)); } else { td::TerminalIO::out() << "unknown command '" << name << "'\n"; } diff --git a/validator-engine-console/validator-engine-console.h b/validator-engine-console/validator-engine-console.h index 7a384276..c802794e 100644 --- a/validator-engine-console/validator-engine-console.h +++ b/validator-engine-console/validator-engine-console.h @@ -57,9 +57,23 @@ class ValidatorEngineConsole : public td::actor::Actor { std::unique_ptr make_callback(); std::map> query_runners_; + std::map alternate_names_; + static std::string simplify_name(std::string name) { + std::erase_if(name, [](char c) { return c == '-'; }); + return name; + } void add_query_runner(std::unique_ptr runner) { auto name = runner->name(); query_runners_[name] = std::move(runner); + alternate_names_[simplify_name(name)] = name; + } + QueryRunner* get_query(std::string name) { + auto it = alternate_names_.find(name); + if (it != alternate_names_.end()) { + name = it->second; + } + auto it2 = query_runners_.find(name); + return it2 == query_runners_.end() ? nullptr : it2->second.get(); } public: diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 78d3ba2b..cc7c57b3 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1503,17 +1503,6 @@ td::Status ValidatorEngine::load_global_config() { h.push_back(b); } validator_options_.write().set_hardforks(std::move(h)); - - auto r_total_mem_stat = td::get_total_mem_stat(); - if (r_total_mem_stat.is_error()) { - LOG(ERROR) << "Failed to get total RAM size: " << r_total_mem_stat.move_as_error(); - } else { - td::uint64 total_ram = r_total_mem_stat.ok().total_ram; - LOG(WARNING) << "Total RAM = " << td::format::as_size(total_ram); - if (total_ram >= (90ULL << 30)) { - fast_state_serializer_enabled_ = true; - } - } validator_options_.write().set_fast_state_serializer_enabled(fast_state_serializer_enabled_); return td::Status::OK(); @@ -3333,6 +3322,70 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_sign &que std::move(query.data_), std::move(P)); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_exportAllPrivateKeys &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_unsafe)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started keyring"))); + return; + } + + ton::PublicKey client_pubkey = ton::PublicKey{query.encryption_key_}; + if (!client_pubkey.is_ed25519()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::protoviolation, "encryption key is not Ed25519"))); + return; + } + + td::actor::send_closure( + keyring_, &ton::keyring::Keyring::export_all_private_keys, + [promise = std::move(promise), + client_pubkey = std::move(client_pubkey)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + // Private keys are encrypted using client-provided public key to avoid storing them in + // non-secure buffers (not td::SecureString) + std::vector serialized_keys; + size_t data_size = 32; + for (const ton::PrivateKey &key : R.ok()) { + serialized_keys.push_back(key.export_as_slice()); + data_size += serialized_keys.back().size() + 4; + } + td::SecureString data{data_size}; + td::MutableSlice slice = data.as_mutable_slice(); + for (const td::SecureString &s : serialized_keys) { + td::uint32 size = td::narrow_cast_safe(s.size()).move_as_ok(); + CHECK(slice.size() >= size + 4); + slice.copy_from(td::Slice{reinterpret_cast(&size), 4}); + slice.remove_prefix(4); + slice.copy_from(s.as_slice()); + slice.remove_prefix(s.size()); + } + CHECK(slice.size() == 32); + td::Random::secure_bytes(slice); + + auto r_encryptor = client_pubkey.create_encryptor(); + if (r_encryptor.is_error()) { + promise.set_value(create_control_query_error(r_encryptor.move_as_error_prefix("cannot create encryptor: "))); + return; + } + auto encryptor = r_encryptor.move_as_ok(); + auto r_encrypted = encryptor->encrypt(data.as_slice()); + if (r_encryptor.is_error()) { + promise.set_value(create_control_query_error(r_encrypted.move_as_error_prefix("cannot encrypt data: "))); + return; + } + promise.set_value(ton::create_serialize_tl_object( + r_encrypted.move_as_ok())); + }); +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setVerbosity &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { @@ -4489,7 +4542,7 @@ int main(int argc, char *argv[]) { }); p.add_option( '\0', "fast-state-serializer", - "faster persistent state serializer, but requires more RAM (enabled automatically on machines with >= 90GB RAM)", + "faster persistent state serializer, but requires more RAM", [&]() { acts.push_back( [&x]() { td::actor::send_closure(x, &ValidatorEngine::set_fast_state_serializer_enabled, true); }); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 65af6178..b7abb0b1 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -475,6 +475,8 @@ class ValidatorEngine : public td::actor::Actor { td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_sign &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_exportAllPrivateKeys &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_setVerbosity &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getStats &query, td::BufferSlice data, ton::PublicKeyHash src, diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index b7f3787c..5f5544b2 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -46,6 +46,7 @@ set(VALIDATOR_HEADERS interfaces/db.h interfaces/external-message.h interfaces/liteserver.h + interfaces/out-msg-queue-proof.h interfaces/proof.h interfaces/shard.h interfaces/signature-set.h diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 9c1ba3c3..7d33a195 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -37,6 +37,7 @@ #include "net/download-proof.hpp" #include "net/get-next-key-blocks.hpp" #include "net/download-archive-slice.hpp" +#include "impl/out-msg-queue-proof.hpp" #include "td/utils/Random.h" @@ -669,6 +670,62 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod query.offset_, query.max_size_, std::move(promise)); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, + td::Promise promise) { + std::vector blocks; + for (const auto &x : query.blocks_) { + BlockIdExt id = create_block_id(x); + if (!id.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid block_id")); + return; + } + if (!shard_is_ancestor(shard_, id.shard_full())) { + promise.set_error(td::Status::Error("query in wrong overlay")); + return; + } + blocks.push_back(create_block_id(x)); + } + ShardIdFull dst_shard = create_shard_id(query.dst_shard_); + if (!dst_shard.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid shard")); + return; + } + block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; + if (limits.max_msgs > 512) { + promise.set_error(td::Status::Error("max_msgs is too big")); + return; + } + if (limits.max_bytes > (1 << 21)) { + promise.set_error(td::Status::Error("max_bytes is too big")); + return; + } + FLOG(DEBUG) { + sb << "Got query getOutMsgQueueProof to shard " << dst_shard.to_str() << " from blocks"; + for (const BlockIdExt &id : blocks) { + sb << " " << id.id.to_str(); + } + sb << " from " << src; + }; + td::actor::send_closure( + full_node_, &FullNode::get_out_msg_queue_query_token, + [=, manager = validator_manager_, blocks = std::move(blocks), + promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, token, std::move(R)); + auto P = + td::PromiseCreator::lambda([promise = std::move(promise), token = std::move(token)]( + td::Result> R) mutable { + if (R.is_error()) { + promise.set_result(create_serialize_tl_object()); + } else { + promise.set_result(serialize_tl_object(R.move_as_ok(), true)); + } + }); + td::actor::create_actor("buildqueueproof", dst_shard, std::move(blocks), limits, manager, + std::move(P)) + .release(); + }); +} + void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { if (!active_) { @@ -944,6 +1001,47 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFu .release(); } +void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) { + // TODO: maybe more complex download (like other requests here) + auto &b = choose_neighbour(); + if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); + return; + } + std::vector> blocks_tl; + for (const BlockIdExt &id : blocks) { + blocks_tl.push_back(create_tl_block_id(id)); + } + td::BufferSlice query = create_serialize_tl_object( + create_tl_shard_id(dst_shard), std::move(blocks_tl), + create_tl_object(limits.max_bytes, limits.max_msgs)); + + auto P = td::PromiseCreator::lambda( + [=, promise = std::move(promise), blocks = std::move(blocks)](td::Result R) mutable { + if (R.is_error()) { + promise.set_result(R.move_as_error()); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { + promise.set_error(td::Status::Error("node doesn't have this block")); + }, + [&](ton_api::tonNode_outMsgQueueProof &x) { + delay_action( + [=, promise = std::move(promise), blocks = std::move(blocks), x = std::move(x)]() mutable { + promise.set_result(OutMsgQueueProof::fetch(dst_shard, blocks, limits, x)); + }, + td::Timestamp::now()); + })); + }); + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_, + "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 22, rldp_); +} + void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise promise) { CHECK(!handle_); handle_ = std::move(handle); diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index f8f500f3..16945325 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -66,6 +66,9 @@ class FullNodeShard : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) = 0; virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 0ae26a7c..86748134 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -139,8 +139,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); - // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, - // td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, + td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); @@ -183,6 +183,9 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise> promise) override; void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) override; void set_handle(BlockHandle handle, td::Promise promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index c1173e44..658cb34e 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -21,6 +21,7 @@ #include "td/actor/MultiPromise.h" #include "full-node.h" #include "common/delay.h" +#include "impl/out-msg-queue-proof.hpp" #include "td/utils/Random.h" #include "ton/ton-tl.hpp" @@ -430,6 +431,24 @@ void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull sh timeout, std::move(promise)); } +void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) { + if (blocks.empty()) { + promise.set_value({}); + return; + } + // All blocks are expected to have the same minsplit shard prefix + auto shard = get_shard(blocks[0].shard_full()); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping download msg queue query to unknown shard"; + promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); + return; + } + td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits, + timeout, std::move(promise)); +} + td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { if (shard.is_masterchain()) { return shards_[ShardIdFull{masterchainId}].actor.get(); @@ -557,6 +576,11 @@ void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, Catcha std::move(data)); } +void FullNodeImpl::get_out_msg_queue_query_token(td::Promise> promise) { + td::actor::send_closure(out_msg_queue_query_token_manager_, &TokenManager::get_token, 1, 0, td::Timestamp::in(10.0), + std::move(promise)); +} + void FullNodeImpl::set_validator_telemetry_filename(std::string value) { validator_telemetry_filename_ = std::move(value); update_validator_telemetry_collector(); @@ -645,6 +669,12 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, shard_prefix, std::move(tmp_dir), timeout, std::move(promise)); } + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits, + timeout, std::move(promise)); + } void new_key_block(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); diff --git a/validator/full-node.h b/validator/full-node.h index 73ecbd72..fdb1bf3b 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -91,6 +91,7 @@ class FullNode : public td::actor::Actor { virtual void process_block_broadcast(BlockBroadcast broadcast) = 0; virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) = 0; + virtual void get_out_msg_queue_query_token(td::Promise> promise) = 0; virtual void set_validator_telemetry_filename(std::string value) = 0; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index b82dd473..0ea6fa0b 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -28,6 +28,7 @@ #include #include #include +#include namespace ton { @@ -79,6 +80,9 @@ class FullNodeImpl : public FullNode { void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise); + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise); void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); @@ -87,6 +91,7 @@ class FullNodeImpl : public FullNode { void process_block_broadcast(BlockBroadcast broadcast) override; void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) override; + void get_out_msg_queue_query_token(td::Promise> promise) override; void set_validator_telemetry_filename(std::string value) override; @@ -160,6 +165,9 @@ class FullNodeImpl : public FullNode { PublicKeyHash validator_telemetry_collector_key_ = PublicKeyHash::zero(); void update_validator_telemetry_collector(); + + td::actor::ActorOwn out_msg_queue_query_token_manager_ = + td::actor::create_actor("tokens", /* max_tokens = */ 1); }; } // namespace fullnode diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index b8f2edfb..978cf859 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -16,6 +16,7 @@ set(TON_VALIDATOR_SOURCE ihr-message.cpp liteserver.cpp message-queue.cpp + out-msg-queue-proof.cpp proof.cpp shard.cpp signature-set.cpp @@ -35,6 +36,7 @@ set(TON_VALIDATOR_SOURCE liteserver.hpp liteserver-cache.hpp message-queue.hpp + out-msg-queue-proof.hpp proof.hpp shard.hpp signature-set.hpp diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp new file mode 100644 index 00000000..95ad4a41 --- /dev/null +++ b/validator/impl/out-msg-queue-proof.cpp @@ -0,0 +1,294 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "out-msg-queue-proof.hpp" +#include "interfaces/proof.h" +#include "shard.hpp" +#include "vm/cells/MerkleProof.h" +#include "common/delay.h" +#include "interfaces/validator-manager.h" +#include "block/block-parse.h" +#include "block/block-auto.h" +#include "output-queue-merger.h" + +namespace ton { + +namespace validator { + +static td::Status check_no_prunned(const Ref& cell) { + if (cell.is_null()) { + return td::Status::OK(); + } + TRY_RESULT(loaded_cell, cell->load_cell()); + if (loaded_cell.data_cell->get_level() > 0) { + return td::Status::Error("prunned branch"); + } + return td::Status::OK(); +} + +static td::Status check_no_prunned(const vm::CellSlice& cs) { + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_STATUS(check_no_prunned(cs.prefetch_ref(i))); + } + return td::Status::OK(); +} + +static td::Result> process_queue( + ShardIdFull dst_shard, std::vector> blocks, + block::ImportedMsgQueueLimits limits) { + td::uint64 estimated_proof_size = 0; + + td::HashSet visited; + std::function dfs_cs; + auto dfs = [&](const Ref& cell) { + if (cell.is_null() || !visited.insert(cell->get_hash()).second) { + return; + } + dfs_cs(vm::CellSlice(vm::NoVm(), cell)); + }; + dfs_cs = [&](const vm::CellSlice& cs) { + // Based on BlockLimitStatus::estimate_block_size + estimated_proof_size += 12 + (cs.size() + 7) / 8 + cs.size_refs() * 3; + for (unsigned i = 0; i < cs.size_refs(); i++) { + dfs(cs.prefetch_ref(i)); + } + }; + std::vector neighbors; + for (auto& b : blocks) { + TRY_STATUS_PREFIX(check_no_prunned(*b.second.proc_info), "invalid proc_info proof: ") + dfs_cs(*b.second.proc_info); + neighbors.emplace_back(b.first, b.second.out_queue->prefetch_ref()); + } + + block::OutputQueueMerger queue_merger{dst_shard, std::move(neighbors)}; + std::vector msg_count(blocks.size()); + td::int32 msg_count_total = 0; + bool limit_reached = false; + + while (!queue_merger.is_eof()) { + auto kv = queue_merger.extract_cur(); + queue_merger.next(); + block::EnqueuedMsgDescr enq; + auto msg = kv->msg; + if (!enq.unpack(msg.write())) { + return td::Status::Error("cannot unpack EnqueuedMsgDescr"); + } + if (limit_reached) { + break; + } + ++msg_count[kv->source]; + ++msg_count_total; + + dfs_cs(*kv->msg); + TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") + if (estimated_proof_size >= limits.max_bytes || msg_count_total >= (long long)limits.max_msgs) { + limit_reached = true; + } + } + if (!limit_reached) { + std::fill(msg_count.begin(), msg_count.end(), -1); + } + return msg_count; +} + +td::Result> OutMsgQueueProof::build( + ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits) { + if (!dst_shard.is_valid_ext()) { + return td::Status::Error("invalid shard"); + } + if (blocks.empty()) { + return create_tl_object(td::BufferSlice{}, td::BufferSlice{}, + std::vector{}); + } + + std::vector> block_state_proofs; + for (auto& block : blocks) { + if (block.id.seqno() != 0) { + if (block.block_root.is_null()) { + return td::Status::Error("block is null"); + } + TRY_RESULT(proof, create_block_state_proof(block.block_root)); + block_state_proofs.push_back(std::move(proof)); + } + if (!block::ShardConfig::is_neighbor(dst_shard, block.id.shard_full())) { + return td::Status::Error("shards are not neighbors"); + } + } + TRY_RESULT(block_state_proof, vm::std_boc_serialize_multi(block_state_proofs)); + + vm::Dictionary states_dict_pure{32}; + for (size_t i = 0; i < blocks.size(); ++i) { + if (blocks[i].state_root.is_null()) { + return td::Status::Error("state is null"); + } + states_dict_pure.set_ref(td::BitArray<32>{(long long)i}, blocks[i].state_root); + } + + vm::MerkleProofBuilder mpb{states_dict_pure.get_root_cell()}; + vm::Dictionary states_dict{mpb.root(), 32}; + std::vector> data(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) { + data[i].first = blocks[i].id; + TRY_RESULT(state, ShardStateQ::fetch(blocks[i].id, {}, states_dict.lookup_ref(td::BitArray<32>{(long long)i}))); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) { + return td::Status::Error("invalid message queue"); + } + } + TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits)); + + TRY_RESULT(proof, mpb.extract_proof()); + vm::Dictionary states_dict_proof{vm::CellSlice{vm::NoVm(), proof}.prefetch_ref(), 32}; + std::vector> state_proofs; + for (size_t i = 0; i < blocks.size(); ++i) { + td::Ref proof_raw = states_dict_proof.lookup_ref(td::BitArray<32>{(long long)i}); + CHECK(proof_raw.not_null()); + state_proofs.push_back(vm::CellBuilder::create_merkle_proof(proof_raw)); + } + TRY_RESULT(queue_proof, vm::std_boc_serialize_multi(state_proofs)); + return create_tl_object(std::move(queue_proof), std::move(block_state_proof), + std::move(msg_count)); +} + +td::Result>> OutMsgQueueProof::fetch(ShardIdFull dst_shard, + std::vector blocks, + block::ImportedMsgQueueLimits limits, + const ton_api::tonNode_outMsgQueueProof& f) { + try { + std::vector> res; + TRY_RESULT(queue_proofs, vm::std_boc_deserialize_multi(f.queue_proofs_, (int)blocks.size())); + TRY_RESULT(block_state_proofs, vm::std_boc_deserialize_multi(f.block_state_proofs_, (int)blocks.size())); + if (queue_proofs.size() != blocks.size()) { + return td::Status::Error("invalid size of queue_proofs"); + } + if (f.msg_counts_.size() != blocks.size()) { + return td::Status::Error("invalid size of msg_counts"); + } + size_t j = 0; + std::vector> data(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) { + td::Bits256 state_root_hash; + Ref block_state_proof = {}; + if (blocks[i].seqno() == 0) { + state_root_hash = blocks[i].root_hash; + } else { + if (j == block_state_proofs.size()) { + return td::Status::Error("invalid size of block_state_proofs"); + } + block_state_proof = block_state_proofs[j++]; + TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(blocks[i], block_state_proof)); + } + auto state_root = vm::MerkleProof::virtualize(queue_proofs[i], 1); + if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { + return td::Status::Error("state root hash mismatch"); + } + res.emplace_back(true, blocks[i], state_root, block_state_proof, f.msg_counts_[i]); + + data[i].first = blocks[i]; + TRY_RESULT(state, ShardStateQ::fetch(blocks[i], {}, state_root)); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) { + return td::Status::Error("invalid message queue"); + } + } + if (j != block_state_proofs.size()) { + return td::Status::Error("invalid size of block_state_proofs"); + } + TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits)); + if (msg_count != f.msg_counts_) { + return td::Status::Error("incorrect msg_count"); + } + return res; + } catch (vm::VmVirtError& err) { + return td::Status::Error(PSTRING() << "invalid proof: " << err.get_msg()); + } +} + +void BuildOutMsgQueueProof::abort_query(td::Status reason) { + if (promise_) { + FLOG(DEBUG) { + sb << "failed to build msg queue proof to " << dst_shard_.to_str() << " from"; + for (const auto& block : blocks_) { + sb << " " << block.id.id.to_str(); + } + sb << ": " << reason; + }; + promise_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof to " << dst_shard_.to_str() << ": ")); + } + stop(); +} + +void BuildOutMsgQueueProof::start_up() { + for (size_t i = 0; i < blocks_.size(); ++i) { + BlockIdExt id = blocks_[i].id; + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db_short, id, + [SelfId = actor_id(this), i](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get shard state: ")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, i, + R.move_as_ok()->root_cell()); + } + }); + if (id.seqno() != 0) { + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_data_from_db_short, id, + [SelfId = actor_id(this), i](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get block data: ")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, i, + R.move_as_ok()->root_cell()); + } + }); + } + } + if (pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::got_state_root(size_t i, Ref root) { + blocks_[i].state_root = std::move(root); + if (--pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::got_block_root(size_t i, Ref root) { + blocks_[i].block_root = std::move(root); + if (--pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::build_proof() { + auto result = OutMsgQueueProof::build(dst_shard_, std::move(blocks_), limits_); + if (result.is_error()) { + LOG(ERROR) << "Failed to build msg queue proof: " << result.error(); + } + promise_.set_result(std::move(result)); + stop(); +} + +} // namespace validator +} // namespace ton diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp new file mode 100644 index 00000000..e28561e2 --- /dev/null +++ b/validator/impl/out-msg-queue-proof.hpp @@ -0,0 +1,64 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" +#include "interfaces/out-msg-queue-proof.h" +#include "td/actor/actor.h" +#include "interfaces/shard.h" +#include "validator.h" + +namespace ton { + +namespace validator { +using td::Ref; + +class ValidatorManager; +class ValidatorManagerInterface; + +class BuildOutMsgQueueProof : public td::actor::Actor { + public: + BuildOutMsgQueueProof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::actor::ActorId manager, + td::Promise> promise) + : dst_shard_(dst_shard), limits_(limits), manager_(manager), promise_(std::move(promise)) { + blocks_.resize(blocks.size()); + for (size_t i = 0; i < blocks_.size(); ++i) { + blocks_[i].id = blocks[i]; + } + } + + void abort_query(td::Status reason); + void start_up() override; + void got_state_root(size_t i, Ref root); + void got_block_root(size_t i, Ref root); + void build_proof(); + + private: + ShardIdFull dst_shard_; + std::vector blocks_; + block::ImportedMsgQueueLimits limits_; + + td::actor::ActorId manager_; + td::Promise> promise_; + + size_t pending = 0; +}; + +} // namespace validator +} // namespace ton diff --git a/validator/impl/proof.cpp b/validator/impl/proof.cpp index 033a1ab1..d7222211 100644 --- a/validator/impl/proof.cpp +++ b/validator/impl/proof.cpp @@ -162,5 +162,40 @@ td::Result> ProofQ::get_signatures_root() const { return proof.signatures->prefetch_ref(); } +td::Result> create_block_state_proof(td::Ref root) { + if (root.is_null()) { + return td::Status::Error("root is null"); + } + vm::MerkleProofBuilder mpb{std::move(root)}; + block::gen::Block::Record block; + if (!tlb::unpack_cell(mpb.root(), block) || block.state_update->load_cell().is_error()) { + return td::Status::Error("invalid block"); + } + TRY_RESULT(proof, mpb.extract_proof()); + if (proof.is_null()) { + return td::Status::Error("failed to create proof"); + } + return proof; +} + +td::Result unpack_block_state_proof(BlockIdExt block_id, td::Ref proof) { + auto virt_root = vm::MerkleProof::virtualize(proof, 1); + if (virt_root.is_null()) { + return td::Status::Error("invalid Merkle proof"); + } + if (virt_root->get_hash().as_slice() != block_id.root_hash.as_slice()) { + return td::Status::Error("hash mismatch"); + } + block::gen::Block::Record block; + if (!tlb::unpack_cell(virt_root, block)) { + return td::Status::Error("invalid block"); + } + vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update}; + if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 && upd_cs.size_ext() == 0x20228)) { + return td::Status::Error("invalid Merkle update"); + } + return upd_cs.prefetch_ref(1)->get_hash(0).bits(); +} + } // namespace validator } // namespace ton diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h new file mode 100644 index 00000000..c0aa5610 --- /dev/null +++ b/validator/interfaces/out-msg-queue-proof.h @@ -0,0 +1,57 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" +#include "block/block.h" + +namespace ton { + +namespace validator { +using td::Ref; + +struct OutMsgQueueProof : public td::CntObject { + OutMsgQueueProof(BlockIdExt block_id, Ref state_root, Ref block_state_proof, + td::int32 msg_count = -1) + : block_id_(block_id) + , state_root_(std::move(state_root)) + , block_state_proof_(std::move(block_state_proof)) + , msg_count_(msg_count) { + } + + BlockIdExt block_id_; + Ref state_root_; + Ref block_state_proof_; + td::int32 msg_count_; // -1 - no limit + + static td::Result>> fetch(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + const ton_api::tonNode_outMsgQueueProof &f); + + struct OneBlock { + BlockIdExt id; + Ref state_root; + Ref block_root; + }; + static td::Result> build(ShardIdFull dst_shard, + std::vector blocks, + block::ImportedMsgQueueLimits limits); +}; + +} // namespace validator +} // namespace ton diff --git a/validator/interfaces/proof.h b/validator/interfaces/proof.h index 99471a1f..6665ad08 100644 --- a/validator/interfaces/proof.h +++ b/validator/interfaces/proof.h @@ -48,6 +48,9 @@ class Proof : virtual public ProofLink { virtual td::Result> export_as_proof_link() const = 0; }; +td::Result> create_block_state_proof(td::Ref root); +td::Result unpack_block_state_proof(BlockIdExt block_id, td::Ref proof); + } // namespace validator } // namespace ton diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 3a568ba4..20d4bd62 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -146,6 +146,9 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast, int mode) = 0; virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; + virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) = 0; virtual void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 18649ba8..cd06bf55 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -265,6 +265,11 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { } + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override { + UNREACHABLE(); + } void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override { UNREACHABLE(); @@ -283,7 +288,7 @@ class ValidatorManagerImpl : public ValidatorManager { void try_get_static_file(FileHash file_hash, td::Promise promise) override; void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { + td::Promise> promise) override { promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 7cd8c705..0b8b9e73 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -335,6 +335,11 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { } + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override { + UNREACHABLE(); + } void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override { UNREACHABLE(); @@ -357,7 +362,7 @@ class ValidatorManagerImpl : public ValidatorManager { void try_get_static_file(FileHash file_hash, td::Promise promise) override; void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { + td::Promise> promise) override { promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); } diff --git a/validator/manager.cpp b/validator/manager.cpp index a75de0a9..a631bd09 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1652,6 +1652,13 @@ void ValidatorManagerImpl::send_validator_telemetry(PublicKeyHash key, callback_->send_validator_telemetry(key, std::move(telemetry)); } +void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( + ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Promise>> promise) { + callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(10.0), + std::move(promise)); +} + void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) { diff --git a/validator/manager.hpp b/validator/manager.hpp index d55e3442..519cab12 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -512,6 +512,9 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override; void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override; + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override; void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; @@ -524,8 +527,8 @@ class ValidatorManagerImpl : public ValidatorManager { void try_get_static_file(FileHash file_hash, td::Promise promise) override; void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { - td::actor::send_closure(token_manager_, &TokenManager::get_download_token, download_size, priority, timeout, + td::Promise> promise) override { + td::actor::send_closure(token_manager_, &TokenManager::get_token, download_size, priority, timeout, std::move(promise)); } diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp index e9a193b4..37580cef 100644 --- a/validator/net/download-block-new.cpp +++ b/validator/net/download-block-new.cpp @@ -144,7 +144,7 @@ void DownloadBlockNew::got_block_handle(BlockHandle handle) { return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -156,7 +156,7 @@ void DownloadBlockNew::got_block_handle(BlockHandle handle) { std::move(P)); } -void DownloadBlockNew::got_download_token(std::unique_ptr token) { +void DownloadBlockNew::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && client_.empty()) { diff --git a/validator/net/download-block-new.hpp b/validator/net/download-block-new.hpp index d2a0e136..ecd062ee 100644 --- a/validator/net/download-block-new.hpp +++ b/validator/net/download-block-new.hpp @@ -49,7 +49,7 @@ class DownloadBlockNew : public td::actor::Actor { void start_up() override; void got_block_handle(BlockHandle handle); - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_data(td::BufferSlice data); void got_data_from_db(td::BufferSlice data); @@ -79,7 +79,7 @@ class DownloadBlockNew : public td::actor::Actor { bool allow_partial_proof_ = false; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/net/download-block.cpp b/validator/net/download-block.cpp index 9ca84be2..c60955ed 100644 --- a/validator/net/download-block.cpp +++ b/validator/net/download-block.cpp @@ -128,7 +128,7 @@ void DownloadBlock::got_block_handle(BlockHandle handle) { return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadBlock::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -140,7 +140,7 @@ void DownloadBlock::got_block_handle(BlockHandle handle) { std::move(P)); } -void DownloadBlock::got_download_token(std::unique_ptr token) { +void DownloadBlock::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && !short_ && client_.empty()) { diff --git a/validator/net/download-block.hpp b/validator/net/download-block.hpp index b1847d58..2e2a715b 100644 --- a/validator/net/download-block.hpp +++ b/validator/net/download-block.hpp @@ -49,7 +49,7 @@ class DownloadBlock : public td::actor::Actor { void start_up() override; void got_block_handle(BlockHandle handle); - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_proof_description(td::BufferSlice proof_description); void got_block_proof(td::BufferSlice data); @@ -86,7 +86,7 @@ class DownloadBlock : public td::actor::Actor { bool allow_partial_proof_ = false; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/net/download-proof.cpp b/validator/net/download-proof.cpp index 2ff95b88..784ecac2 100644 --- a/validator/net/download-proof.cpp +++ b/validator/net/download-proof.cpp @@ -107,7 +107,7 @@ void DownloadProof::start_up() { } void DownloadProof::checked_db() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadProof::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -119,7 +119,7 @@ void DownloadProof::checked_db() { std::move(P)); } -void DownloadProof::got_download_token(std::unique_ptr token) { +void DownloadProof::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && client_.empty()) { diff --git a/validator/net/download-proof.hpp b/validator/net/download-proof.hpp index 0739dcaf..9caf9c3a 100644 --- a/validator/net/download-proof.hpp +++ b/validator/net/download-proof.hpp @@ -45,7 +45,7 @@ class DownloadProof : public td::actor::Actor { void start_up() override; void checked_db(); - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_proof_description(td::BufferSlice proof_description); void got_block_proof(td::BufferSlice data); @@ -72,7 +72,7 @@ class DownloadProof : public td::actor::Actor { td::BufferSlice data_; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/net/get-next-key-blocks.cpp b/validator/net/get-next-key-blocks.cpp index 3354b005..2c12e495 100644 --- a/validator/net/get-next-key-blocks.cpp +++ b/validator/net/get-next-key-blocks.cpp @@ -84,7 +84,7 @@ void GetNextKeyBlocks::finish_query() { void GetNextKeyBlocks::start_up() { alarm_timestamp() = timeout_; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &GetNextKeyBlocks::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -96,7 +96,7 @@ void GetNextKeyBlocks::start_up() { std::move(P)); } -void GetNextKeyBlocks::got_download_token(std::unique_ptr token) { +void GetNextKeyBlocks::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && client_.empty()) { diff --git a/validator/net/get-next-key-blocks.hpp b/validator/net/get-next-key-blocks.hpp index 074289e2..14c040bd 100644 --- a/validator/net/get-next-key-blocks.hpp +++ b/validator/net/get-next-key-blocks.hpp @@ -44,7 +44,7 @@ class GetNextKeyBlocks : public td::actor::Actor { void finish_query(); void start_up() override; - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void send_request(); void got_result(td::BufferSlice res); @@ -75,7 +75,7 @@ class GetNextKeyBlocks : public td::actor::Actor { std::vector pending_; std::vector res_; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/token-manager.cpp b/validator/token-manager.cpp index 0bc4a9c6..8242f921 100644 --- a/validator/token-manager.cpp +++ b/validator/token-manager.cpp @@ -22,23 +22,23 @@ namespace ton { namespace validator { -void TokenManager::get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { +void TokenManager::get_token(size_t size, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) { if (free_priority_tokens_ > 0 && priority > 0) { --free_priority_tokens_; - promise.set_value(gen_token(download_size, priority)); + promise.set_value(gen_token(size, priority)); return; } if (free_tokens_ > 0) { --free_tokens_; - promise.set_value(gen_token(download_size, priority)); + promise.set_value(gen_token(size, priority)); return; } - pending_.emplace(PendingPromiseKey{download_size, priority, seqno_++}, PendingPromise{timeout, std::move(promise)}); + pending_.emplace(PendingPromiseKey{size, priority, seqno_++}, PendingPromise{timeout, std::move(promise)}); } -void TokenManager::download_token_cleared(size_t download_size, td::uint32 priority) { +void TokenManager::token_cleared(size_t size, td::uint32 priority) { (priority ? free_priority_tokens_ : free_tokens_)++; if (free_priority_tokens_ > max_priority_tokens_) { free_priority_tokens_--; @@ -47,7 +47,7 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior for (auto it = pending_.begin(); it != pending_.end();) { if (it->first.priority && (free_tokens_ || free_priority_tokens_)) { - it->second.promise.set_value(gen_token(download_size, priority)); + it->second.promise.set_value(gen_token(size, priority)); auto it2 = it++; pending_.erase(it2); if (free_priority_tokens_ > 0) { @@ -56,7 +56,7 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior free_tokens_--; } } else if (!it->first.priority && free_tokens_) { - it->second.promise.set_value(gen_token(download_size, priority)); + it->second.promise.set_value(gen_token(size, priority)); auto it2 = it++; pending_.erase(it2); free_tokens_--; @@ -69,7 +69,7 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior void TokenManager::alarm() { for (auto it = pending_.begin(); it != pending_.end();) { if (it->second.timeout.is_in_past()) { - it->second.promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout in wait download token")); + it->second.promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout in wait token")); it = pending_.erase(it); } else { it++; @@ -77,23 +77,23 @@ void TokenManager::alarm() { } } -std::unique_ptr TokenManager::gen_token(size_t download_size, td::uint32 priority) { - class Token : public DownloadToken { +std::unique_ptr TokenManager::gen_token(size_t size, td::uint32 priority) { + class TokenImpl : public ActionToken { public: - Token(size_t download_size, td::uint32 priority, td::actor::ActorId manager) - : download_size_(download_size), priority_(priority), manager_(manager) { + TokenImpl(size_t size, td::uint32 priority, td::actor::ActorId manager) + : size_(size), priority_(priority), manager_(manager) { } - ~Token() override { - td::actor::send_closure(manager_, &TokenManager::download_token_cleared, download_size_, priority_); + ~TokenImpl() override { + td::actor::send_closure(manager_, &TokenManager::token_cleared, size_, priority_); } private: - size_t download_size_; + size_t size_; td::uint32 priority_; td::actor::ActorId manager_; }; - return std::make_unique(download_size, priority, actor_id(this)); + return std::make_unique(size, priority, actor_id(this)); } } // namespace validator diff --git a/validator/token-manager.h b/validator/token-manager.h index 0d75710f..0fd0126a 100644 --- a/validator/token-manager.h +++ b/validator/token-manager.h @@ -31,16 +31,19 @@ class TokenManager : public td::actor::Actor { public: TokenManager() { } + explicit TokenManager(td::uint32 max_tokens) + : free_tokens_(max_tokens), free_priority_tokens_(max_tokens), max_priority_tokens_(max_tokens) { + } void alarm() override; - void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise); - void download_token_cleared(size_t download_size, td::uint32 priority); + void get_token(size_t size, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise); + void token_cleared(size_t size, td::uint32 priority); private: - std::unique_ptr gen_token(size_t download_size, td::uint32 priority); + std::unique_ptr gen_token(size_t size, td::uint32 priority); struct PendingPromiseKey { - size_t download_size; + size_t size; td::uint32 priority; td::uint64 seqno; @@ -50,7 +53,7 @@ class TokenManager : public td::actor::Actor { }; struct PendingPromise { td::Timestamp timeout; - td::Promise> promise; + td::Promise> promise; }; td::uint64 seqno_ = 0; std::map pending_; diff --git a/validator/validator.h b/validator/validator.h index cc7cbe62..73065aa9 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -35,15 +35,16 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" #include "catchain/catchain-types.h" +#include "interfaces/out-msg-queue-proof.h" #include "interfaces/external-message.h" namespace ton { namespace validator { -class DownloadToken { +class ActionToken { public: - virtual ~DownloadToken() = default; + virtual ~ActionToken() = default; }; struct PerfTimerStats { @@ -186,6 +187,9 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; @@ -248,7 +252,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void add_ext_server_port(td::uint16 port) = 0; virtual void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) = 0; + td::Promise> promise) = 0; virtual void get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0;