From 9f008b129f1fec6c72a5e67e69ddf9caca02d27f Mon Sep 17 00:00:00 2001 From: ton Date: Mon, 27 Apr 2020 16:01:46 +0400 Subject: [PATCH] updated submodules, bugfixes - added new fift/func code for validator complaint creation - bugfixes in validator - updates in tonlib - new versions of rocksdb/abseil - hardfork support --- .gitignore | 1 - CMakeLists.txt | 7 +- .../blockchain-explorer-query.cpp | 2 +- crypto/CMakeLists.txt | 15 +- crypto/block/block-parse.cpp | 16 + crypto/block/block-parse.h | 5 + crypto/block/block.cpp | 90 +- crypto/block/block.h | 62 +- crypto/block/block.tlb | 30 +- crypto/block/check-proof.cpp | 10 +- crypto/block/check-proof.h | 2 +- crypto/block/mc-config.cpp | 25 + crypto/block/mc-config.h | 6 + crypto/block/test-weight-distr.cpp | 199 ++ crypto/block/transaction.cpp | 3 + crypto/common/bigint.hpp | 20 +- crypto/common/promiseop.hpp | 72 + crypto/common/refint.cpp | 4 +- crypto/common/refint.h | 2 +- crypto/fift/lib/TonUtil.fif | 8 + crypto/smartcont/CreateState.fif | 45 +- crypto/smartcont/complaint-vote-req.fif | 29 + crypto/smartcont/complaint-vote-signed.fif | 44 + crypto/smartcont/elector-code.fc | 27 +- crypto/smartcont/envelope-complaint.fif | 50 + crypto/smartcont/new-restricted-wallet2.fif | 4 +- crypto/smartcont/payment-channel-code.fc | 315 ++++ crypto/smartcont/restricted-wallet2-code.fc | 35 +- crypto/smartcont/restricted-wallet3-code.fc | 103 ++ crypto/smartcont/show-addr.fif | 4 +- crypto/smc-envelope/GenericAccount.cpp | 27 +- crypto/smc-envelope/GenericAccount.h | 4 + crypto/smc-envelope/HighloadWallet.cpp | 10 +- crypto/smc-envelope/HighloadWalletV2.cpp | 11 +- crypto/smc-envelope/PaymentChannel.cpp | 264 +++ crypto/smc-envelope/PaymentChannel.h | 251 +++ crypto/smc-envelope/SmartContract.cpp | 35 +- crypto/smc-envelope/SmartContract.h | 14 + crypto/smc-envelope/SmartContractCode.cpp | 23 +- crypto/smc-envelope/SmartContractCode.h | 14 +- crypto/smc-envelope/TestGiver.cpp | 7 +- crypto/smc-envelope/TestWallet.cpp | 11 +- crypto/smc-envelope/Wallet.cpp | 10 +- crypto/smc-envelope/WalletInterface.h | 22 +- crypto/smc-envelope/WalletV3.cpp | 10 +- crypto/smc-envelope/WalletV3.h | 206 +++ crypto/test/test-db.cpp | 1 + crypto/test/test-smartcont.cpp | 490 +++++ crypto/vm/cells/CellSlice.cpp | 9 + crypto/vm/cells/CellSlice.h | 2 + crypto/vm/cells/MerkleProof.cpp | 18 + crypto/vm/cells/MerkleProof.h | 2 + crypto/vm/db/TonDb.cpp | 6 +- crypto/vm/excno.hpp | 33 +- lite-client/lite-client.cpp | 1601 +++++++++++++++-- lite-client/lite-client.h | 153 +- tdactor/td/actor/PromiseFuture.h | 174 +- tdactor/td/actor/actor.h | 22 + tdactor/td/actor/common.h | 34 + tdactor/td/actor/core/CpuWorker.cpp | 2 + tdactor/td/actor/core/IoWorker.cpp | 4 + tdactor/td/actor/core/Scheduler.cpp | 17 +- tdactor/td/actor/core/Scheduler.h | 59 +- tdactor/td/actor/core/SchedulerContext.h | 4 + tdactor/test/actors_core.cpp | 18 +- tdactor/test/actors_promise.cpp | 76 +- tddb/td/db/KeyValue.h | 14 + tddb/td/db/MemoryKeyValue.cpp | 9 + tddb/td/db/MemoryKeyValue.h | 4 + tddb/td/db/RocksDb.cpp | 49 +- tddb/td/db/RocksDb.h | 4 + tdutils/td/utils/AtomicRead.h | 90 + tdutils/td/utils/Status.h | 10 + tdutils/test/StealingQueue.cpp | 41 + test/test-rocksdb.cpp | 97 + third-party/abseil-cpp | 2 +- third-party/rocksdb | 2 +- tl/generate/scheme/tonlib_api.tl | 46 +- tl/generate/scheme/tonlib_api.tlo | Bin 21792 -> 25548 bytes tonlib/test/online.cpp | 194 +- tonlib/tonlib/Client.cpp | 1 + tonlib/tonlib/Config.cpp | 20 + tonlib/tonlib/Config.h | 2 + tonlib/tonlib/LastBlock.h | 7 +- tonlib/tonlib/TonlibClient.cpp | 638 ++++++- tonlib/tonlib/TonlibClient.h | 10 + tonlib/tonlib/TonlibError.h | 5 +- tonlib/tonlib/tonlib-cli.cpp | 739 ++++++-- validator-engine/validator-engine.cpp | 38 +- validator-engine/validator-engine.hpp | 8 +- .../validator-session-description.cpp | 2 +- validator/CMakeLists.txt | 30 + validator/block-handle.cpp | 4 +- validator/block-handle.hpp | 4 +- validator/db/archive-manager.cpp | 175 +- validator/db/archive-manager.hpp | 5 +- validator/db/archive-slice.cpp | 240 ++- validator/db/archive-slice.hpp | 29 +- validator/db/celldb.cpp | 12 +- validator/db/rootdb.cpp | 9 +- validator/db/rootdb.hpp | 12 +- validator/db/statedb.cpp | 109 +- validator/db/statedb.hpp | 1 + validator/downloaders/wait-block-state.cpp | 72 +- validator/downloaders/wait-block-state.hpp | 2 + validator/fabric.h | 7 +- validator/impl/accept-block.cpp | 6 +- validator/impl/check-proof.cpp | 17 +- validator/impl/collator-impl.h | 7 +- validator/impl/collator.cpp | 88 +- validator/impl/fabric.cpp | 32 +- validator/impl/liteserver.cpp | 177 +- validator/impl/liteserver.hpp | 9 +- validator/impl/validate-query.cpp | 7 +- validator/import-db-slice.cpp | 4 +- validator/interfaces/db.h | 4 +- validator/interfaces/validator-manager.h | 2 +- validator/manager-disk.cpp | 2 +- validator/manager-disk.hpp | 2 +- validator/manager-hardfork.cpp | 563 ++++++ validator/manager-hardfork.h | 35 + validator/manager-hardfork.hpp | 429 +++++ validator/manager-init.cpp | 156 +- validator/manager-init.hpp | 5 +- validator/manager.cpp | 80 +- validator/manager.hpp | 3 +- validator/validator-options.cpp | 8 +- validator/validator-options.hpp | 44 +- validator/validator.h | 15 +- 129 files changed, 8438 insertions(+), 879 deletions(-) create mode 100644 crypto/block/test-weight-distr.cpp create mode 100644 crypto/common/promiseop.hpp create mode 100644 crypto/smartcont/complaint-vote-req.fif create mode 100644 crypto/smartcont/complaint-vote-signed.fif create mode 100644 crypto/smartcont/envelope-complaint.fif create mode 100644 crypto/smartcont/payment-channel-code.fc create mode 100644 crypto/smartcont/restricted-wallet3-code.fc create mode 100644 crypto/smc-envelope/PaymentChannel.cpp create mode 100644 crypto/smc-envelope/PaymentChannel.h create mode 100644 tdutils/td/utils/AtomicRead.h create mode 100644 test/test-rocksdb.cpp create mode 100644 validator/manager-hardfork.cpp create mode 100644 validator/manager-hardfork.h create mode 100644 validator/manager-hardfork.hpp diff --git a/.gitignore b/.gitignore index 44ed769e..3d77c68c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ tl/generate/auto/ compile_commands.json crypto/block/block-auto.cpp crypto/block/block-auto.h -crypto/smartcont/*-code.fif crypto/smartcont/auto/ test/regression-tests.cache/ *.swp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e85633b..941865f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,6 +115,7 @@ if (TON_USE_ROCKSDB) if (ANDROID) set(PORTABLE ON CACHE BOOL "portable") endif() + set(WITH_GFLAGS OFF CACHE BOOL "build with GFlags") set(WITH_TESTS OFF CACHE BOOL "build with tests") set(WITH_TOOLS OFF CACHE BOOL "build with tools") set(FAIL_ON_WARNINGS OFF CACHE BOOL "fail on warnings") @@ -231,7 +232,7 @@ elseif (CLANG OR GCC) if (APPLE) #use "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/export_list" for exported symbols set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S") + #set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,--exclude-libs,ALL") @@ -385,6 +386,7 @@ add_subdirectory(validator) add_subdirectory(blockchain-explorer) add_subdirectory(validator-engine) add_subdirectory(validator-engine-console) +add_subdirectory(create-hardfork) add_subdirectory(dht-server) add_subdirectory(utils) add_subdirectory(http) @@ -445,6 +447,9 @@ if (NOT TON_ONLY_TONLIB) add_executable(test-db test/test-td-main.cpp ${TONDB_TEST_SOURCE}) target_link_libraries(test-db PRIVATE ton_db memprof) +add_executable(test-rocksdb test/test-rocksdb.cpp) +target_link_libraries(test-rocksdb PRIVATE memprof tddb tdutils) + add_executable(test-tddb test/test-td-main.cpp ${TDDB_TEST_SOURCE}) target_link_libraries(test-tddb PRIVATE tdutils tddb ${CMAKE_THREAD_LIBS_INIT} memprof) diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index 4c8ed7b1..b53e7969 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -1505,7 +1505,7 @@ void HttpQueryStatus::finish_query() { td::uint32 j = 0; for (auto &X : results_.results) { if (!X->values_[i].is_valid()) { - A << "FAIL"; + A << "FAIL"; } else { if (m[j].count(X->values_[i].id.seqno) == 0) { m[j].insert(X->values_[i].id.seqno); diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index f2865f55..d8efc586 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -49,6 +49,7 @@ set(TON_CRYPTO_SOURCE common/bigexp.h common/util.h common/linalloc.hpp + common/promiseop.hpp ellcurve/Ed25519.h ellcurve/Fp25519.h @@ -212,6 +213,7 @@ set(SMC_ENVELOPE_SOURCE smc-envelope/HighloadWalletV2.cpp smc-envelope/ManualDns.cpp smc-envelope/MultisigWallet.cpp + smc-envelope/PaymentChannel.cpp smc-envelope/SmartContract.cpp smc-envelope/SmartContractCode.cpp smc-envelope/TestGiver.cpp @@ -341,7 +343,7 @@ if (NOT CMAKE_CROSSCOMPILING) set(multiValueArgs SOURCE) set(FUNC_LIB_SOURCE smartcont/stdlib.fc) cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - string(REGEX REPLACE "[^a-zA-Z_]" "_" ID ${ARG_DEST}) + string(REGEX REPLACE "[^0-9a-zA-Z_]" "_" ID ${ARG_DEST}) set(ARG_DEST_FIF "${ARG_DEST}.fif") add_custom_command( COMMENT "Generate ${ARG_DEST_FIF}" @@ -374,10 +376,13 @@ if (NOT CMAKE_CROSSCOMPILING) GenFif(DEST smartcont/auto/multisig-code SOURCE smartcont/multisig-code.fc NAME multisig) GenFif(DEST smartcont/auto/restricted-wallet-code SOURCE smartcont/restricted-wallet-code.fc NAME restricted-wallet) GenFif(DEST smartcont/auto/restricted-wallet2-code SOURCE smartcont/restricted-wallet2-code.fc NAME restricted-wallet2) + GenFif(DEST smartcont/auto/restricted-wallet3-code SOURCE smartcont/restricted-wallet3-code.fc NAME restricted-wallet3) GenFif(DEST smartcont/auto/dns-manual-code SOURCE smartcont/dns-manual-code.fc NAME dns-manual) GenFif(DEST smartcont/auto/dns-auto-code SOURCE smartcont/dns-auto-code.fc NAME dns-auto) + GenFif(DEST smartcont/auto/payment-channel-code SOURCE smartcont/payment-channel-code.fc NAME payment-channel) + GenFif(DEST smartcont/auto/simple-wallet-ext-code SOURCE smartcont/simple-wallet-ext-code.fc NAME simple-wallet-ext) endif() @@ -420,5 +425,13 @@ if (WINGETOPT_FOUND) target_link_libraries_system(dump-block wingetopt) endif() +add_executable(test-weight-distr block/test-weight-distr.cpp) +target_include_directories(test-weight-distr PUBLIC $ + $) +target_link_libraries(test-weight-distr PUBLIC ton_crypto fift-lib ton_block) +if (WINGETOPT_FOUND) + target_link_libraries_system(test-weight-distr wingetopt) +endif() + install(TARGETS fift func RUNTIME DESTINATION bin) install(DIRECTORY fift/lib/ DESTINATION lib/fift) diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 125d2a19..c62854d4 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -241,6 +241,14 @@ bool MsgAddressInt::extract_std_address(vm::CellSlice& cs, ton::WorkchainId& wor return false; } +bool MsgAddressInt::extract_std_address(Ref cs_ref, block::StdAddress& addr, bool rewrite) const { + return extract_std_address(std::move(cs_ref), addr.workchain, addr.addr, rewrite); +} + +bool MsgAddressInt::extract_std_address(vm::CellSlice& cs, block::StdAddress& addr, bool rewrite) const { + return extract_std_address(cs, addr.workchain, addr.addr, rewrite); +} + bool MsgAddressInt::store_std_address(vm::CellBuilder& cb, ton::WorkchainId workchain, const ton::StdSmcAddress& addr) const { if (workchain >= -128 && workchain < 128) { @@ -263,6 +271,14 @@ Ref MsgAddressInt::pack_std_address(ton::WorkchainId workchain, c } } +bool MsgAddressInt::store_std_address(vm::CellBuilder& cb, const block::StdAddress& addr) const { + return store_std_address(cb, addr.workchain, addr.addr); +} + +Ref MsgAddressInt::pack_std_address(const block::StdAddress& addr) const { + return pack_std_address(addr.workchain, addr.addr); +} + const MsgAddressInt t_MsgAddressInt; bool MsgAddress::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index b2ed9c95..1c490383 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -298,8 +298,13 @@ struct MsgAddressInt final : TLB_Complex { bool rewrite = true) const; bool extract_std_address(vm::CellSlice& cs, ton::WorkchainId& workchain, ton::StdSmcAddress& addr, bool rewrite = true) const; + bool extract_std_address(Ref cs_ref, block::StdAddress& addr, bool rewrite = true) const; + bool extract_std_address(vm::CellSlice& cs, block::StdAddress& addr, bool rewrite = true) const; bool store_std_address(vm::CellBuilder& cb, ton::WorkchainId workchain, const ton::StdSmcAddress& addr) const; Ref pack_std_address(ton::WorkchainId workchain, const ton::StdSmcAddress& addr) const; + + bool store_std_address(vm::CellBuilder& cb, const block::StdAddress& addr) const; + Ref pack_std_address(const block::StdAddress& addr) const; }; extern const MsgAddressInt t_MsgAddressInt; diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index c7b437a6..9e2caef5 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -27,6 +27,7 @@ #include "td/utils/crypto.h" #include "td/utils/tl_storers.h" #include "td/utils/misc.h" +#include "td/utils/Random.h" namespace block { using namespace std::literals::string_literals; @@ -369,14 +370,14 @@ std::unique_ptr MsgProcessedUptoCollection::unpack(t return v && v->valid ? std::move(v) : std::unique_ptr{}; } -bool MsgProcessedUpto::contains(const MsgProcessedUpto& other) const & { +bool MsgProcessedUpto::contains(const MsgProcessedUpto& other) const& { return ton::shard_is_ancestor(shard, other.shard) && mc_seqno >= other.mc_seqno && (last_inmsg_lt > other.last_inmsg_lt || (last_inmsg_lt == other.last_inmsg_lt && !(last_inmsg_hash < other.last_inmsg_hash))); } bool MsgProcessedUpto::contains(ton::ShardId other_shard, ton::LogicalTime other_lt, td::ConstBitPtr other_hash, - ton::BlockSeqno other_mc_seqno) const & { + ton::BlockSeqno other_mc_seqno) const& { return ton::shard_is_ancestor(shard, other_shard) && mc_seqno >= other_mc_seqno && (last_inmsg_lt > other_lt || (last_inmsg_lt == other_lt && !(last_inmsg_hash < other_hash))); } @@ -1546,6 +1547,89 @@ bool unpack_CreatorStats(Ref cs, DiscountedCounter& mc_cnt, Disco } } +/* + * + * Monte Carlo simulator for computing the share of shardchain blocks generated by each validator + * + */ + +bool MtCarloComputeShare::compute() { + ok = false; + if (W.size() >= (1U << 31) || W.empty()) { + return false; + } + K = std::min(K, N); + if (K <= 0 || iterations <= 0) { + return false; + } + double tot_weight = 0., acc = 0.; + for (int i = 0; i < N; i++) { + if (W[i] <= 0.) { + return false; + } + tot_weight += W[i]; + } + CW.resize(N); + RW.resize(N); + for (int i = 0; i < N; i++) { + CW[i] = acc; + acc += W[i] /= tot_weight; + RW[i] = 0.; + } + R0 = 0.; + H.resize(N); + A.resize(K); + for (long long it = 0; it < iterations; ++it) { + gen_vset(); + } + for (int i = 0; i < N; i++) { + RW[i] = W[i] * (RW[i] + R0) / (double)iterations; + } + return ok = true; +} + +void MtCarloComputeShare::gen_vset() { + double total_wt = 1.; + int hc = 0; + for (int i = 0; i < K; i++) { + CHECK(total_wt > 0); + double inv_wt = 1. / total_wt; + R0 += inv_wt; + for (int j = 0; j < i; j++) { + RW[A[j]] -= inv_wt; + } + // double p = drand48() * total_wt; + double p = (double)td::Random::fast_uint64() * total_wt / (1. * (1LL << 32) * (1LL << 32)); + for (int h = 0; h < hc; h++) { + if (p < H[h].first) { + break; + } + p += H[h].second; + } + int a = -1, b = N, c; + while (b - a > 1) { + c = ((a + b) >> 1); + if (CW[c] <= p) { + a = c; + } else { + b = c; + } + } + CHECK(a >= 0 && a < N); + CHECK(total_wt >= W[a]); + total_wt -= W[a]; + double x = CW[a]; + c = hc++; + while (c > 0 && H[c - 1].first > x) { + H[c] = H[c - 1]; + --c; + } + H[c].first = x; + H[c].second = W[a]; + A[i] = a; + } +} + /* * * Other block-related functions @@ -1723,7 +1807,7 @@ ton::AccountIdPrefixFull interpolate_addr(const ton::AccountIdPrefixFull& src, c unsigned long long mask = (std::numeric_limits::max() >> (d - 32)); return ton::AccountIdPrefixFull{dest.workchain, (dest.account_id_prefix & ~mask) | (src.account_id_prefix & mask)}; } else { - int mask = (-1 >> d); + int mask = (int)(~0U >> d); return ton::AccountIdPrefixFull{(dest.workchain & ~mask) | (src.workchain & mask), src.account_id_prefix}; } } diff --git a/crypto/block/block.h b/crypto/block/block.h index 5184a349..6c460e31 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -163,12 +163,12 @@ struct MsgProcessedUpto { MsgProcessedUpto(ton::ShardId _shard, ton::BlockSeqno _mcseqno, ton::LogicalTime _lt, td::ConstBitPtr _hash) : shard(_shard), mc_seqno(_mcseqno), last_inmsg_lt(_lt), last_inmsg_hash(_hash) { } - bool operator<(const MsgProcessedUpto& other) const & { + bool operator<(const MsgProcessedUpto& other) const& { return shard < other.shard || (shard == other.shard && mc_seqno < other.mc_seqno); } - bool contains(const MsgProcessedUpto& other) const &; + bool contains(const MsgProcessedUpto& other) const&; bool contains(ton::ShardId other_shard, ton::LogicalTime other_lt, td::ConstBitPtr other_hash, - ton::BlockSeqno other_mc_seqno) const &; + ton::BlockSeqno other_mc_seqno) const&; // NB: this is for checking whether we have already imported an internal message bool already_processed(const EnqueuedMsgDescr& msg) const; bool can_check_processed() const { @@ -596,6 +596,62 @@ struct BlockProofChain { td::Status validate(td::CancellationToken cancellation_token = {}); }; +// compute the share of shardchain blocks generated by each validator using Monte Carlo method +class MtCarloComputeShare { + int K, N; + long long iterations; + std::vector W; + std::vector CW, RW; + std::vector> H; + std::vector A; + double R0; + bool ok; + + public: + MtCarloComputeShare(int subset_size, const std::vector& weights, long long iteration_count = 1000000) + : K(subset_size), N((int)weights.size()), iterations(iteration_count), W(weights), ok(false) { + compute(); + } + MtCarloComputeShare(int subset_size, int set_size, const double* weights, long long iteration_count = 1000000) + : K(subset_size), N(set_size), iterations(iteration_count), W(weights, weights + set_size), ok(false) { + compute(); + } + bool is_ok() const { + return ok; + } + const double* share_array() const { + return ok ? RW.data() : nullptr; + } + const double* weights_array() const { + return ok ? W.data() : nullptr; + } + double operator[](int i) const { + return ok ? RW.at(i) : -1.; + } + double share(int i) const { + return ok ? RW.at(i) : -1.; + } + double weight(int i) const { + return ok ? W.at(i) : -1.; + } + int size() const { + return N; + } + int subset_size() const { + return K; + } + long long performed_iterations() const { + return iterations; + } + + private: + bool set_error() { + return ok = false; + } + bool compute(); + void gen_vset(); +}; + int filter_out_msg_queue(vm::AugmentedDictionary& out_queue, ton::ShardIdFull old_shard, ton::ShardIdFull subshard); std::ostream& operator<<(std::ostream& os, const ShardId& shard_id); diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 1744a071..f43e1bea 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -272,6 +272,7 @@ transaction$0111 account_addr:bits256 lt:uint64 old:^X new:^X = MERKLE_UPDATE X; update_hashes#72 {X:Type} old_hash:bits256 new_hash:bits256 = HASH_UPDATE X; +!merkle_proof#03 {X:Type} virtual_hash:bits256 depth:uint16 virtual_root:^X = MERKLE_PROOF X; acc_trans#5 account_addr:bits256 transactions:(HashmapAug 64 ^Transaction CurrencyCollection) @@ -731,8 +732,11 @@ top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockD // // VALIDATOR MISBEHAVIOR COMPLAINTS // -no_blk_gen mc_blk_ref:ExtBlkRef from_utime:uint32 to_utime:uint32 state_proof:^Cell prod_proof:^Cell = ComplaintDescr; -validator_complaint#bc validator_pubkey:uint256 description:^ComplaintDescr created_at:uint32 severity:uint8 reward_addr:uint256 paid:Grams suggested_fine:Grams suggested_fine_part:uint32 = ValidatorComplaint; +prod_info#34 utime:uint32 mc_blk_ref:ExtBlkRef state_proof:^(MERKLE_PROOF Block) + prod_proof:^(MERKLE_PROOF ShardState) = ProducerInfo; +no_blk_gen from_utime:uint32 prod_info:^ProducerInfo = ComplaintDescr; +no_blk_gen_diff prod_info_old:^ProducerInfo prod_info_new:^ProducerInfo = ComplaintDescr; +validator_complaint#bc validator_pubkey:bits256 description:^ComplaintDescr created_at:uint32 severity:uint8 reward_addr:uint256 paid:Grams suggested_fine:Grams suggested_fine_part:uint32 = ValidatorComplaint; complaint_status#2d complaint:^ValidatorComplaint voters:(HashmapE 16 True) vset_id:uint256 weight_remaining:int64 = ValidatorComplaintStatus; // @@ -808,3 +812,25 @@ cap_method_pubkey#71f4 = SmcCapability; cap_is_wallet#2177 = SmcCapability; cap_name#ff name:Text = SmcCapability; +// +// PAYMENT CHANNELS +// + +chan_config$_ init_timeout:uint32 close_timeout:uint32 a_key:bits256 b_key:bits256 + a_addr:^MsgAddressInt b_addr:^MsgAddressInt channel_id:uint64 = ChanConfig; + +chan_state_init$000 signed_A:Bool signed_B:Bool min_A:Grams min_B:Grams expire_at:uint32 A:Grams B:Grams = ChanState; +chan_state_close$001 signed_A:Bool signed_B:Bool promise_A:Grams promise_B:Grams expire_at:uint32 A:Grams B:Grams = ChanState; +chan_state_payout$010 A:Grams B:Grams = ChanState; + +chan_promise$_ channel_id:uint64 promise_A:Grams promise_B:Grams = ChanPromise; +chan_signed_promise#_ sig:(Maybe ^bits512) promise:ChanPromise = ChanSignedPromise; + +chan_msg_init#27317822 inc_A:Grams inc_B:Grams min_A:Grams min_B:Grams channel_id:uint64 = ChanMsg; +chan_msg_close#f28ae183 extra_A:Grams extra_B:Grams promise:ChanSignedPromise = ChanMsg; +chan_msg_timeout#43278a28 = ChanMsg; + +chan_signed_msg$_ sig_A:(Maybe ^bits512) sig_B:(Maybe ^bits512) msg:ChanMsg = ChanSignedMsg; + +chan_data$_ config:^ChanConfig state:^ChanState = ChanData; + diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp index 6d991256..6720ad40 100644 --- a/crypto/block/check-proof.cpp +++ b/crypto/block/check-proof.cpp @@ -31,7 +31,7 @@ namespace block { using namespace std::literals::string_literals; -td::Status check_block_header_proof(td::Ref root, ton::BlockIdExt blkid, ton::Bits256* store_shard_hash_to, +td::Status check_block_header_proof(td::Ref root, ton::BlockIdExt blkid, ton::Bits256* store_state_hash_to, bool check_state_hash, td::uint32* save_utime, ton::LogicalTime* save_lt) { ton::RootHash vhash{root->get_hash().bits()}; if (vhash != blkid.root_hash) { @@ -53,7 +53,7 @@ td::Status check_block_header_proof(td::Ref root, ton::BlockIdExt blki if (save_lt) { *save_lt = info.end_lt; } - if (store_shard_hash_to) { + if (store_state_hash_to) { vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update}; if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 // merkle update && upd_cs.size_ext() == 0x20228)) { @@ -61,11 +61,11 @@ td::Status check_block_header_proof(td::Ref root, ton::BlockIdExt blki } auto upd_hash = upd_cs.prefetch_ref(1)->get_hash(0); if (!check_state_hash) { - *store_shard_hash_to = upd_hash.bits(); - } else if (store_shard_hash_to->compare(upd_hash.bits())) { + *store_state_hash_to = upd_hash.bits(); + } else if (store_state_hash_to->compare(upd_hash.bits())) { return td::Status::Error(PSTRING() << "state hash mismatch in block header of " << blkid.to_str() << " : header declares " << upd_hash.bits().to_hex(256) << " expected " - << store_shard_hash_to->to_hex()); + << store_state_hash_to->to_hex()); } } return td::Status::OK(); diff --git a/crypto/block/check-proof.h b/crypto/block/check-proof.h index cfd3a0d2..527f3138 100644 --- a/crypto/block/check-proof.h +++ b/crypto/block/check-proof.h @@ -25,7 +25,7 @@ namespace block { using td::Ref; td::Status check_block_header_proof(td::Ref root, ton::BlockIdExt blkid, - ton::Bits256* store_shard_hash_to = nullptr, bool check_state_hash = false, + ton::Bits256* store_state_hash_to = nullptr, bool check_state_hash = false, td::uint32* save_utime = nullptr, ton::LogicalTime* save_lt = nullptr); td::Status check_shard_proof(ton::BlockIdExt blk, ton::BlockIdExt shard_blk, td::Slice shard_proof); td::Status check_account_proof(td::Slice proof, ton::BlockIdExt shard_blk, const block::StdAddress& addr, diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index e192ad81..7fab4750 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -1785,6 +1785,31 @@ std::vector ValidatorSet::export_validator_set() const { return l; } +std::map ValidatorSet::compute_validator_map() const { + std::map res; + for (int i = 0; i < (int)list.size(); i++) { + res.emplace(list[i].pubkey.as_bits256(), i); + } + return res; +} + +std::vector ValidatorSet::export_scaled_validator_weights() const { + std::vector res; + for (const auto& node : list) { + res.push_back((double)node.weight / (double)total_weight); + } + return res; +} + +int ValidatorSet::lookup_public_key(td::ConstBitPtr pubkey) const { + for (int i = 0; i < (int)list.size(); i++) { + if (list[i].pubkey.as_bits256() == pubkey) { + return i; + } + } + return -1; +} + std::vector Config::do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf, ton::ShardIdFull shard, const block::ValidatorSet& vset, ton::UnixTime time, diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index f51b4262..8be3f6e6 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -71,6 +71,12 @@ struct ValidatorSet { } const ValidatorDescr& at_weight(td::uint64 weight_pos) const; std::vector export_validator_set() const; + std::map compute_validator_map() const; + std::vector export_scaled_validator_weights() const; + int lookup_public_key(td::ConstBitPtr pubkey) const; + int lookup_public_key(const td::Bits256& pubkey) const { + return lookup_public_key(pubkey.bits()); + } }; #pragma pack(push, 1) diff --git a/crypto/block/test-weight-distr.cpp b/crypto/block/test-weight-distr.cpp new file mode 100644 index 00000000..a94b791c --- /dev/null +++ b/crypto/block/test-weight-distr.cpp @@ -0,0 +1,199 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2020 Telegram Systems LLP +*/ +#include +#include "td/utils/Random.h" +#include "td/utils/misc.h" +#include "block/block.h" +#include + +const int MAX_N = 1000, MAX_K = 100, DEFAULT_K = 7; + +int verbosity; +int N, K = DEFAULT_K; +long long iterations = 1000000; + +td::uint64 TWL, WL[MAX_N]; +double W[MAX_N], CW[MAX_N + 1], RW[MAX_N], R0; +int A[MAX_N], C[MAX_N]; +long long TC; + +void gen_vset() { + static std::pair H[MAX_N]; + double total_wt = 1.; + int hc = 0; + for (int i = 0; i < K; i++) { + CHECK(total_wt > 0); + double inv_wt = 1. / total_wt; + R0 += inv_wt; // advanced mtcarlo stats + for (int j = 0; j < i; j++) { + RW[A[j]] -= inv_wt; // advanced mtcarlo stats + } + // double p = drand48() * total_wt; + double p = (double)td::Random::fast_uint64() * total_wt / (1. * (1LL << 32) * (1LL << 32)); + for (int h = 0; h < hc; h++) { + if (p < H[h].first) { + break; + } + p += H[h].second; + } + int a = -1, b = N, c; + while (b - a > 1) { + c = ((a + b) >> 1); + if (CW[c] <= p) { + a = c; + } else { + b = c; + } + } + CHECK(a >= 0 && a < N); + CHECK(total_wt >= W[a]); + total_wt -= W[a]; + double x = CW[a]; + c = hc++; + while (c > 0 && H[c - 1].first > x) { + H[c] = H[c - 1]; + --c; + } + H[c].first = x; + H[c].second = W[a]; + A[i] = a; + C[a]++; // simple mtcarlo stats + // std::cout << a << ' '; + } + // std::cout << std::endl; + ++TC; // simple mtcarlo stats +} + +void mt_carlo() { + for (int i = 0; i < N; i++) { + C[i] = 0; + RW[i] = 0.; + } + TC = 0; + R0 = 0.; + std::cout << "running " << iterations << " steps of Monte Carlo simulation\n"; + for (long long it = 0; it < iterations; ++it) { + gen_vset(); + } + for (int i = 0; i < N; i++) { + RW[i] = W[i] * (RW[i] + R0) / (double)iterations; + } +} + +double B[MAX_N]; + +void compute_bad_approx() { + static double S[MAX_K + 1]; + S[0] = 1.; + for (int i = 1; i <= K; i++) { + S[i] = 0.; + } + for (int i = 0; i < N; i++) { + double p = W[i]; + for (int j = K; j > 0; j--) { + S[j] += p * S[j - 1]; + } + } + double Sk = S[K]; + for (int i = 0; i < N; i++) { + double t = 1., p = W[i]; + for (int j = 1; j <= K; j++) { + t = S[j] - p * t; + } + B[i] = 1. - t / Sk; + } +} + +void usage() { + std::cout + << "usage: test-weight-distr [-k][-m][-s]\nReads the set of validator " + "weights from stdin and emulates validator shard distribution load\n\t-k \tSets the number of " + "validators generating each shard\n\t-m \tMonte Carlo simulation steps\n"; + std::exit(2); +} + +int main(int argc, char* const argv[]) { + int i; + int new_verbosity_level = VERBOSITY_NAME(INFO); + // long seed = 0; + while ((i = getopt(argc, argv, "hs:k:m:v:")) != -1) { + switch (i) { + case 'k': + K = td::to_integer(td::Slice(optarg)); + CHECK(K > 0 && K <= 100); + break; + case 'm': + iterations = td::to_integer(td::Slice(optarg)); + CHECK(iterations > 0); + break; + case 's': + // seed = td::to_integer(td::Slice(optarg)); + // srand48(seed); + break; + case 'v': + new_verbosity_level = VERBOSITY_NAME(FATAL) + (verbosity = td::to_integer(td::Slice(optarg))); + break; + case 'h': + usage(); + std::exit(2); + default: + usage(); + std::exit(2); + } + } + SET_VERBOSITY_LEVEL(new_verbosity_level); + for (N = 0; N < MAX_N && (std::cin >> WL[N]); N++) { + CHECK(WL[N] > 0); + TWL += WL[N]; + } + CHECK(std::cin.eof()); + CHECK(N > 0 && TWL > 0 && N <= MAX_N); + K = std::min(K, N); + CHECK(K > 0 && K <= MAX_K); + double acc = 0.; + for (i = 0; i < N; i++) { + CW[i] = acc; + acc += W[i] = (double)WL[i] / (double)TWL; + std::cout << "#" << i << ":\t" << W[i] << std::endl; + } + compute_bad_approx(); + mt_carlo(); + std::cout << "result of Monte Carlo simulation (" << iterations << " iterations):" << std::endl; + std::cout << "idx\tweight\tmtcarlo1\tmtcarlo2\tapprox\n"; + for (i = 0; i < N; i++) { + std::cout << "#" << i << ":\t" << W[i] << '\t' << (double)C[i] / (double)iterations << '\t' << RW[i] << '\t' << B[i] + << std::endl; + } + // same computation, but using a MtCarloComputeShare object + block::MtCarloComputeShare MT(K, N, W, iterations); + std::cout << "-----------------------\n"; + for (i = 0; i < N; i++) { + std::cout << '#' << i << ":\t" << MT.weight(i) << '\t' << MT.share(i) << std::endl; + } + return 0; +} diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 0e3ad5f2..3f5a1ae6 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -1429,6 +1429,9 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (!tlb::csr_unpack(msg.info, erec)) { return -1; } + if (act_rec.mode & ~3) { + return -1; // invalid mode for an external message + } info.src = std::move(erec.src); info.dest = std::move(erec.dest); // created_lt and created_at are ignored diff --git a/crypto/common/bigint.hpp b/crypto/common/bigint.hpp index 7f6cf971..cde80ce2 100644 --- a/crypto/common/bigint.hpp +++ b/crypto/common/bigint.hpp @@ -272,7 +272,7 @@ class AnyIntView { int parse_binary_any(const char* str, int str_len, int* frac = nullptr); std::string to_dec_string_destroy_any(); std::string to_dec_string_slow_destroy_any(); - std::string to_hex_string_any(bool upcase = false) const; + std::string to_hex_string_any(bool upcase = false, int zero_pad = 0) const; std::string to_hex_string_slow_destroy_any(); std::string to_binary_string_any() const; @@ -650,7 +650,7 @@ class BigIntG { std::string to_dec_string_destroy(); std::string to_dec_string_slow() const; std::string to_hex_string_slow() const; - std::string to_hex_string(bool upcase = false) const; + std::string to_hex_string(bool upcase = false, int zero_pad = 0) const; std::string to_binary_string() const; double to_double() const { return is_valid() ? ldexp(top_double(), (n - 1) * word_shift) : NAN; @@ -2290,16 +2290,19 @@ std::string AnyIntView::to_hex_string_slow_destroy_any() { } template -std::string AnyIntView::to_hex_string_any(bool upcase) const { +std::string AnyIntView::to_hex_string_any(bool upcase, int zero_pad) const { if (!is_valid()) { return "NaN"; } int s = sgn(), k = 0; if (!s) { + if (zero_pad > 0) { + return std::string(zero_pad, '0'); + } return "0"; } std::string x; - x.reserve(((size() * word_shift + word_bits) >> 2) + 2); + x.reserve(2 + std::max((size() * word_shift + word_bits) >> 2, zero_pad)); assert(word_shift < word_bits - 4); const char* hex_digs = (upcase ? HEX_digits : hex_digits); word_t v = 0; @@ -2317,6 +2320,11 @@ std::string AnyIntView::to_hex_string_any(bool upcase) const { x += hex_digs[v & 15]; v >>= 4; } + if (zero_pad > 0) { + while (x.size() < (unsigned)zero_pad) { + x += '0'; + } + } if (s < 0) { x += '-'; } @@ -2498,8 +2506,8 @@ std::string BigIntG::to_hex_string_slow() const { } template -std::string BigIntG::to_hex_string(bool upcase) const { - return as_any_int().to_hex_string_any(upcase); +std::string BigIntG::to_hex_string(bool upcase, int zero_pad) const { + return as_any_int().to_hex_string_any(upcase, zero_pad); } template diff --git a/crypto/common/promiseop.hpp b/crypto/common/promiseop.hpp new file mode 100644 index 00000000..1e14bc7d --- /dev/null +++ b/crypto/common/promiseop.hpp @@ -0,0 +1,72 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2020 Telegram Systems LLP +*/ +#pragma once +#include "refcnt.hpp" +#include "td/actor/PromiseFuture.h" + +namespace td { + +template +class BinaryPromiseMerger : public CntObject { + Result first_; + Result second_; + Promise> promise_; + std::atomic pending_; + + public: + BinaryPromiseMerger(Promise> promise) : promise_(std::move(promise)), pending_(2) { + } + static std::pair, Promise> split(Promise> promise) { + auto ref = make_ref(std::move(promise)); + auto& obj = ref.write(); + return std::make_pair(obj.left(), obj.right()); + } + + private: + Promise left() { + return [this, self = Ref(this)](Result res) { + first_ = std::move(res); + work(); + }; + } + Promise right() { + return [this, self = Ref(this)](Result res) { + second_ = std::move(res); + work(); + }; + } + void work() { + if (!--pending_) { + if (first_.is_error()) { + promise_.set_error(first_.move_as_error()); + } else if (second_.is_error()) { + promise_.set_error(second_.move_as_error()); + } else { + promise_.set_result(std::pair(first_.move_as_ok(), second_.move_as_ok())); + } + } + } +}; + +template +std::pair, Promise> split_promise(Promise> promise) { + return BinaryPromiseMerger::split(std::move(promise)); +} + +} // namespace td diff --git a/crypto/common/refint.cpp b/crypto/common/refint.cpp index a8fe1762..b79750ce 100644 --- a/crypto/common/refint.cpp +++ b/crypto/common/refint.cpp @@ -287,8 +287,8 @@ std::string dec_string2(RefInt256&& x) { return x.is_null() ? "(null)" : (x.is_unique() ? x.unique_write().to_dec_string_destroy() : x->to_dec_string()); } -std::string hex_string(RefInt256 x, bool upcase) { - return x.is_null() ? "(null)" : x->to_hex_string(upcase); +std::string hex_string(RefInt256 x, bool upcase, int zero_pad) { + return x.is_null() ? "(null)" : x->to_hex_string(upcase, zero_pad); } std::string binary_string(RefInt256 x) { diff --git a/crypto/common/refint.h b/crypto/common/refint.h index a905eb77..488ae97f 100644 --- a/crypto/common/refint.h +++ b/crypto/common/refint.h @@ -120,7 +120,7 @@ extern RefInt256 bits_to_refint(td::ConstBitPtr bits, int n, bool sgnd = false); extern std::string dec_string(RefInt256 x); extern std::string dec_string2(RefInt256&& x); -extern std::string hex_string(RefInt256 x, bool upcase = false); +extern std::string hex_string(RefInt256 x, bool upcase = false, int zero_pad = 0); extern std::string binary_string(RefInt256 x); extern RefInt256 dec_string_to_int256(const std::string& s); diff --git a/crypto/fift/lib/TonUtil.fif b/crypto/fift/lib/TonUtil.fif index bcdb4a7e..1b978a28 100644 --- a/crypto/fift/lib/TonUtil.fif +++ b/crypto/fift/lib/TonUtil.fif @@ -36,6 +36,7 @@ library TonUtil // TON Blockchain Fift Library // ( x -- ) Displays a 64-digit hex number { 64 0x. } : 64x. +{ 64 0X. } : 64X. // ( wc addr -- ) Show address in : form { swap ._ .":" 64x. } : .addr // ( wc addr flags -- ) Show address in base64url form @@ -205,6 +206,13 @@ recursive append-long-bytes { { 256 u>B B{3ee6} swap B+ dup crc16 16 u>B B+ B>base64 } : pubkey>$ { pubkey>$ type } : .pubkey +// ( S -- x ) parse validator-encoded public key +{ base64>B dup Blen 36 <> abort"public key with magic must be 36 bytes long" + 4 B| swap 32 B>u@ 0xC6B41348 <> abort"unknown magic for public key (not Ed25519)" +} : parse-val-pubkey +{ bl word parse-val-pubkey 1 'nop } ::_ VPK' +{ char } word base64>B 1 'nop } ::_ B64{ + // adnl address parser { 256 u>B B{2D} swap B+ dup crc16 16 u>B B+ } : adnl-preconv { swap 32 /mod dup 26 < { 65 } { 24 } cond + rot swap hold } : Base32# diff --git a/crypto/smartcont/CreateState.fif b/crypto/smartcont/CreateState.fif index 7866a840..0b2ee9f7 100644 --- a/crypto/smartcont/CreateState.fif +++ b/crypto/smartcont/CreateState.fif @@ -62,15 +62,31 @@ variable validators-weight validators-weight 0! { validator-dict @ second } : validator# -// val-pubkey weight -- { dup 0<= abort"validator weight must be non-negative" - dup 64 ufits not abort"validator weight must fit into 64 bits" + 64 ufits not abort"validator weight must fit into 64 bits" +} : check-val-weight +// ( val-pubkey weight -- c ) +{ dup check-val-weight over Blen 32 <> abort"validator public key must be 32 bytes long" - tuck +} : serialize-validator +// ( val-pubkey adnl weight -- c ) +{ dup check-val-weight + over 256 ufits not abort"adnl address must fit into 256 bits" + rot dup Blen 32 <> abort"validator public key must be 32 bytes long" + +} : serialize-adnl-validator +// ( weight val-cell -- ) +{ swap validators-weight +! + idict! not abort"cannot add value" +{ idict! not abort"cannot add value" } : rdict-entry +{ 86400 * } : days* // balance -- dict { dictnew - over -32768 rdict-entry - over 3/4 */ 92 rdict-entry - over 1/2 */ 183 rdict-entry - swap 1/4 */ 366 rdict-entry - 0 548 rdict-entry + over 31 -1<< rdict-entry + over 3/4 */ 91 days* rdict-entry + over 1/2 */ 183 days* rdict-entry + swap 1/4 */ 365 days* rdict-entry + 0 548 days* rdict-entry } : make-rdict +variable wallet2-start-at wallet2-start-at 0! +now 86400 / 1+ 86400 * wallet2-start-at ! // pubkey amount -- { over ."Key " pubkey>$ type ." -> " RWCode2 // code - // data + // data empty_cell // libs 3 roll // balance 0 // split_depth diff --git a/crypto/smartcont/complaint-vote-req.fif b/crypto/smartcont/complaint-vote-req.fif new file mode 100644 index 00000000..f771f6fd --- /dev/null +++ b/crypto/smartcont/complaint-vote-req.fif @@ -0,0 +1,29 @@ +#!/usr/bin/fift -s +"TonUtil.fif" include +"GetOpt.fif" include + +"validator-to-sign.req" =: savefile + +{ ."usage: " @' $0 type ." []" cr + ."Creates an unsigned request to vote for complaint (decimal; prefix with '0x' if needed) of past validator set on behalf of validator with zero-based index in current validator set (as stored in configuration parameter 34)." cr + ."The result is saved into (" savefile type ." by default) and output in hexadecimal form, to be signed later by the validator public key" cr 1 halt +} : usage + +$# dup 3 < swap 4 > or ' usage if +4 :$1..n + +$1 parse-int dup =: val-idx + 16 ufits not abort"validator index out of range" +$2 parse-int dup =: elect-id + 32 ufits not abort"invalid election id" +$3 parse-int dup =: compl-hash + 256 ufits not abort"invalid complaint hash" +$4 savefile replace-if-null =: savefile + +."Creating a request to vote for complaint 0x" compl-hash 64x. ."of past validator set " elect-id . +."on behalf of current validator with index " val-idx . cr + +B{56744350} val-idx 16 u>B B+ elect-id 32 u>B B+ compl-hash 256 u>B B+ +dup Bx. cr +dup B>base64url type cr +savefile tuck B>file ."Saved to file " type cr diff --git a/crypto/smartcont/complaint-vote-signed.fif b/crypto/smartcont/complaint-vote-signed.fif new file mode 100644 index 00000000..90a36a54 --- /dev/null +++ b/crypto/smartcont/complaint-vote-signed.fif @@ -0,0 +1,44 @@ +#!/usr/bin/fift -s +"TonUtil.fif" include +"GetOpt.fif" include + +"vote-query.boc" =: savefile + +{ ."usage: " @' $0 type ." []" cr + ."Creates an internal message body to be sent from any smart contract residing in the masterchain to the elections smart contract containing a signed request to vote for complaint (decimal; prefix with '0x' if needed) of past validator set on behalf of current validator with zero-based index and (Base64) public key in current validator set (as stored in configuration parameter 34)" cr + ." must be the base64 representation of Ed25519 signature of the previously generated unsigned request by means of " cr + ."The result is saved into (" savefile type ." by default), to be embedded later into an internal message" cr 1 halt +} : usage + +$# dup 5 < swap 6 > or ' usage if +6 :$1..n + +$1 parse-int dup =: val-idx + 16 ufits not abort"validator index out of range" +$2 parse-int dup =: elect-id + 32 ufits not abort"invalid election id" +$3 parse-int dup =: compl-hash + 256 ufits not abort"invalid complaint hash" +$4 base64>B dup Blen 36 <> abort"validator Ed25519 public key must be exactly 36 bytes long" + 32 B>u@+ 0xC6B41348 <> abort"invalid Ed25519 public key: unknown magic number" + =: pubkey +$5 base64>B dup Blen 64 <> abort"validator Ed25519 signature must be exactly 64 bytes long" + =: signature +$6 savefile replace-if-null =: savefile + +."Creating the body of an internal message to be sent to the elections smart contract" cr } +."containing a signed request to vote for complaint 0x" compl-hash 64x. ."of past validator set " elect-id . +."on behalf of current validator with index " val-idx . "and public key" pubkey Bx. cr + +B{56744350} val-idx 16 u>B B+ elect-id 32 u>B B+ compl-hash 256 u>B B+ dup =: to_sign +."String to sign is " Bx. cr + +to_sign signature pubkey ed25519_chksign not abort"Ed25519 signature is invalid" +."Provided a valid Ed25519 signature " signature Bx. ." with validator public key " pubkey Bx. cr + +now 32 << compl-hash 32 1<< mod + =: query-id + + +."Internal message body is " dup B savefile tuck B>file ."Saved to file " type cr diff --git a/crypto/smartcont/elector-code.fc b/crypto/smartcont/elector-code.fc index 33436800..844ac90a 100644 --- a/crypto/smartcont/elector-code.fc +++ b/crypto/smartcont/elector-code.fc @@ -1097,8 +1097,21 @@ tuple past_elections() method_id { do { (id, var fs, var found) = past_elections.udict_get_prev?(32, id); if (found) { - var info = [unpack_past_election(fs)]; - list = cons(pair(id, info), list); + list = cons([id, unpack_past_election(fs)], list); + } + } until (~ found); + return list; +} + +tuple past_elections_list() method_id { + var (elect, credits, past_elections, grams, active_id, active_hash) = load_data(); + var id = (1 << 32); + var list = null(); + do { + (id, var fs, var found) = past_elections.udict_get_prev?(32, id); + if (found) { + var (unfreeze_at, stake_held, vset_hash, frozen_dict, total_stake, bonuses, complaints) = unpack_past_election(fs); + list = cons([id, unfreeze_at, vset_hash, stake_held], list); } } until (~ found); return list; @@ -1117,7 +1130,7 @@ _ complete_unpack_complaint(slice cs) inline_ref { return [[complaint.begin_parse().unpack_complaint()], voters_list, vset_id, weight_remaining]; } -cell get_past_complaints(int election_id) inline_ref { +cell get_past_complaints(int election_id) inline_ref method_id { var (elect, credits, past_elections, grams, active_id, active_hash) = load_data(); var (fs, found?) = past_elections.udict_get?(32, election_id); ifnot (found?) { @@ -1145,3 +1158,11 @@ tuple list_complaints(int election_id) method_id { } until (~ found?); return list; } + +int complaint_storage_price(int bits, int refs, int expire_in) method_id { + ;; compute complaint storage/creation price + var (deposit, bit_price, cell_price) = get_complaint_prices(); + var pps = (bits + 1024) * bit_price + (refs + 2) * cell_price; + var paid = pps * expire_in + deposit; + return paid + (1 << 30); +} diff --git a/crypto/smartcont/envelope-complaint.fif b/crypto/smartcont/envelope-complaint.fif new file mode 100644 index 00000000..fa7b857e --- /dev/null +++ b/crypto/smartcont/envelope-complaint.fif @@ -0,0 +1,50 @@ +#!/usr/bin/fift -s +"TonUtil.fif" include +"GetOpt.fif" include + +{ show-options-help 1 halt } : usage +86400 3 * =: expire-in +false =: critical +-1 =: old-hash + +begin-options + " [-x ] []" +cr +tab + +"Embeds a validator complaint loaded from file into an internal message body to be sent to the elector smart contract " + +"and saves it as an internal message body into .boc ('complaint-msg-body.boc' by default)" + disable-digit-options generic-help-setopt + "x" "--expires-in" { parse-int =: expire-in } short-long-option-arg + "Sets complaint expiration time in seconds (default " expire-in (.) $+ +")" option-help + "h" "--help" { usage } short-long-option + "Shows a help message" option-help +parse-options + +$# dup 2 < swap 3 > or ' usage if +3 :$1..n + +$1 parse-int dup =: election-id + 32 ufits not abort"invalid election id" +$2 =: boc-filename +$3 "complaint-msg-body.boc" replace-if-null =: savefile +expire-in now + =: expire-at + +boc-filename dup ."Loading complaint from file `" type ."`" cr +file>B B>boc dup =: complaint hash =: c-hash +complaint abort"Not a valid ValidatorComplaint" +complaint + +dup ."resulting internal message body: " B dup Bx. cr + +complaint +totalcsize swap ."(a total of " . ."data bits, " . ."cell references -> " +drop dup Blen . ."BoC data bytes)" cr + +savefile tuck B>file +."(Saved to file " type .")" cr diff --git a/crypto/smartcont/new-restricted-wallet2.fif b/crypto/smartcont/new-restricted-wallet2.fif index 8f95e4c1..90f54495 100644 --- a/crypto/smartcont/new-restricted-wallet2.fif +++ b/crypto/smartcont/new-restricted-wallet2.fif @@ -29,8 +29,8 @@ def? $3 { @' $3 } { "rwallet" } cond constant file-base } : make-rdict // Create new restricted wallet; code taken from `auto/restricted-wallet2-code.fif` -"auto/restricted-wallet-code.fif" include // code - // data +"auto/restricted-wallet2-code.fif" include // code + // data null // no libraries // create StateInit dup ."StateInit: " b_sig:Maybe msg:Message = SignedMessage; +;; checks signatures and unwap message. +(slice, (int, int)) unwrap_signatures(slice cs, int a_key, int b_key) { + int a? = cs~load_int(1); + slice a_sig = cs; + if (a?) { + a_sig = cs~load_ref().begin_parse().preload_bits(512); + } + var b? = cs~load_int(1); + slice b_sig = cs; + if (b?) { + b_sig = cs~load_ref().begin_parse().preload_bits(512); + } + int hash = cs.slice_hash(); + if (a?) { + throw_unless(err:wrong_a_signature(), check_signature(hash, a_sig, a_key)); + } + if (b?) { + throw_unless(err:wrong_b_signature(), check_signature(hash, b_sig, b_key)); + } + return (cs, (a?, b?)); +} + +;; process message, give state is stateInit +;; +;; stateInit signed_A?:Bool signed_B?:Bool min_A:Grams min_B:Grams expire_at:uint32 A:Grams B:Grams = State; +_ unpack_state_init(slice state) { + return ( + state~load_int(1), + state~load_int(1), + state~load_grams(), + state~load_grams(), + state~load_uint(32), + state~load_grams(), + state~load_grams()); + +} +_ pack_state_init(int signed_A?, int signed_B?, int min_A, int min_B, int expire_at, int A, int B) { + return begin_cell() + .store_int(state:init(), 3) + .store_int(signed_A?, 1) + .store_int(signed_B?, 1) + .store_grams(min_A) + .store_grams(min_B) + .store_uint(expire_at, 32) + .store_grams(A) + .store_grams(B).end_cell(); +} + +;; stateClosing$10 signed_A?:bool signed_B?:Bool promise_A:Grams promise_B:Grams exipire_at:uint32 A:Grams B:Grams = State; +_ unpack_state_close(slice state) { + return ( + state~load_int(1), + state~load_int(1), + state~load_grams(), + state~load_grams(), + state~load_uint(32), + state~load_grams(), + state~load_grams()); +} + +_ pack_state_close(int signed_A?, int signed_B?, int promise_A, int promise_B, int expire_at, int A, int B) { + return begin_cell() + .store_int(state:close(), 3) + .store_int(signed_A?, 1) + .store_int(signed_B?, 1) + .store_grams(promise_A) + .store_grams(promise_B) + .store_uint(expire_at, 32) + .store_grams(A) + .store_grams(B).end_cell(); +} + +_ send_payout(slice s_addr, int amount, int channel_id, int flags) impure { + send_raw_message(begin_cell() + .store_uint(0x10, 6) + .store_slice(s_addr) + .store_grams(amount) + .store_uint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) + .store_uint(msg:payout(), 32) + .store_uint(channel_id, 64) + .end_cell(), flags); +} + + +cell do_payout(int promise_A, int promise_B, int A, int B, slice a_addr, slice b_addr, int channel_id) impure { + accept_message(); + + int diff = promise_B - promise_A; + if (diff < - A) { + diff = - A; + } + if (diff > B) { + diff = B; + } + A += diff; + B -= diff; + + send_payout(a_addr, A, channel_id, 3); + send_payout(b_addr, B, channel_id, 3); + + return begin_cell() + .store_int(state:payout(), 3) + .store_grams(A) + .store_grams(B) + .end_cell(); +} + + +;; +;; init$000 inc_A:Grams inc_B:Grams min_A:Grams min_B:Grams = Message; +;; +cell with_init(slice state, int msg_value, slice msg, int msg_signed_A?, int msg_signed_B?, + slice a_addr, slice b_addr, int init_timeout, int channel_id) { + ;; parse state + (int signed_A?, int signed_B?, int min_A, int min_B, int expire_at, int A, int B) = unpack_state_init(state); + + if (expire_at == 0) { + expire_at = now() + init_timeout; + } + + int op = msg~load_uint(32); + if (op == msg:timeout()) { + throw_unless(err:no_timeout(), expire_at < now()); + return do_payout(0, 0, A, B, a_addr, b_addr, channel_id); + } + throw_unless(err:expected_init(), op == msg:init()); + + ;; unpack init message + (int inc_A, int inc_B, int upd_min_A, int upd_min_B, int got_channel_id) = + (msg~load_grams(), msg~load_grams(), msg~load_grams(), msg~load_grams(), msg~load_uint(64)); + throw_unless(err:wrong_channel_id(), got_channel_id == channel_id); + + ;; TODO: we should reserve some part of the value for comission + throw_if(err:msg_value_too_small(), msg_value < inc_A + inc_B); + throw_unless(err:replay_protection(), (msg_signed_A? < signed_A?) | (msg_signed_B? < signed_B?)); + + A += inc_A; + B += inc_B; + + signed_A? |= msg_signed_A?; + if (min_A < upd_min_A) { + min_A = upd_min_A; + } + + signed_B? |= msg_signed_B?; + if (min_B < upd_min_B) { + min_B = upd_min_B; + } + + if (signed_A? & signed_B?) { + if ((min_A > A) | (min_B > B)) { + return do_payout(0, 0, A, B, a_addr, b_addr, channel_id); + } + + return pack_state_close(0, 0, 0, 0, 0, A, B); + } + + return pack_state_init(signed_A?, signed_B?, min_A, min_B, expire_at, A, B); +} + +;; close$001 extra_A:Grams extra_B:Grams sig:Maybe promise_A:Grams promise_B:Grams = Message; + +cell with_close(slice cs, slice msg, int msg_signed_A?, int msg_signed_B?, int a_key, int b_key, + slice a_addr, slice b_addr, int expire_timeout, int channel_id) { + ;; parse state + (int signed_A?, int signed_B?, int promise_A, int promise_B, int expire_at, int A, int B) = unpack_state_close(cs); + + if (expire_at == 0) { + expire_at = now() + expire_timeout; + } + + int op = msg~load_uint(32); + if (op == msg:timeout()) { + throw_unless(err:no_timeout(), expire_at < now()); + return do_payout(promise_A, promise_B, A, B, a_addr, b_addr, channel_id); + } + throw_unless(err:expected_close(), op == msg:close()); + + ;; also ensures that (msg_signed_A? | msg_signed_B?) is true + throw_unless(err:replay_protection(), (msg_signed_A? < signed_A?) | (msg_signed_B? < signed_B?)); + signed_A? |= msg_signed_A?; + signed_B? |= msg_signed_B?; + + ;; unpack close message + (int extra_A, int extra_B) = (msg~load_grams(), msg~load_grams()); + int has_sig = msg~load_int(1); + if (has_sig) { + slice sig = msg~load_ref().begin_parse().preload_bits(512); + int hash = msg.slice_hash(); + ifnot (msg_signed_A?) { + throw_unless(err:wrong_a_signature(), check_signature(hash, sig, a_key)); + extra_A = 0; + } + ifnot (msg_signed_B?) { + throw_unless(err:wrong_b_signature(), check_signature(hash, sig, b_key)); + extra_B = 0; + } + } else { + throw_unless(err:no_promise_signature(), msg_signed_A? & msg_signed_B?); + extra_A = 0; + extra_B = 0; + } + (int got_channel_id, int update_promise_A, int update_promise_B) = (msg~load_uint(64), msg~load_grams(), msg~load_grams()); + throw_unless(err:wrong_channel_id(), got_channel_id == channel_id); + + + accept_message(); + update_promise_A += extra_A; + if (promise_A < update_promise_A) { + promise_A = update_promise_A; + } + update_promise_B += extra_B; + if (promise_B < update_promise_B) { + promise_B = update_promise_B; + } + + if (signed_A? & signed_B?) { + return do_payout(promise_A, promise_B, A, B, a_addr, b_addr, channel_id); + } + return pack_state_close(signed_A?, signed_B?, promise_A, promise_B, expire_at, A, B); +} + +() recv_any(int msg_value, slice msg) impure { + (cell config, cell state) = unpack_data(); + (int init_timeout, int close_timeout, int a_key, int b_key, slice a_addr, slice b_addr, int channel_id) = config.unpack_config(); + (int msg_signed_A?, int msg_signed_B?) = msg~unwrap_signatures(a_key, b_key); + + slice cs = state.begin_parse(); + int state_type = cs~load_uint(3); + + if (state_type == state:init()) { ;; init + state = with_init(cs, msg_value, msg, msg_signed_A?, msg_signed_B?, a_addr, b_addr, init_timeout, channel_id); + } if (state_type == state:close()) { + state = with_close(cs, msg, msg_signed_A?, msg_signed_B?, a_key, b_key, a_addr, b_addr, close_timeout, channel_id); + } + + pack_data(config, state); +} + +() recv_internal(int msg_value, cell in_msg_cell, slice in_msg) impure { + recv_any(msg_value, in_msg); +} + +() recv_external(slice in_msg) impure { + recv_any(0, in_msg); +} diff --git a/crypto/smartcont/restricted-wallet2-code.fc b/crypto/smartcont/restricted-wallet2-code.fc index 8d509886..fbad6c09 100644 --- a/crypto/smartcont/restricted-wallet2-code.fc +++ b/crypto/smartcont/restricted-wallet2-code.fc @@ -1,13 +1,16 @@ ;; Restricted wallet (a variant of wallet-code.fc) -;; until configuration parameter -13 is set, accepts messages only to elector smc +;; restricts access to parts of balance until certain dates () recv_internal(slice in_msg) impure { ;; do nothing for internal messages } -_ days_passed() inline { - var p = config_param(-13); - return null?(p) ? -1 : (now() - begin_parse(p).preload_uint(32)) / 86400; +_ seconds_passed(int start_at, int utime) inline_ref { + ifnot (start_at) { + var p = config_param(-13); + start_at = null?(p) ? 0 : begin_parse(p).preload_uint(32); + } + return start_at ? utime - start_at : -1; } () recv_external(slice in_msg) impure { @@ -16,7 +19,7 @@ _ days_passed() inline { var (msg_seqno, valid_until) = (cs~load_uint(32), cs~load_uint(32)); throw_if(35, valid_until <= now()); var ds = get_data().begin_parse(); - var (stored_seqno, public_key, rdict) = (ds~load_uint(32), ds~load_uint(256), ds~load_dict()); + var (stored_seqno, public_key, start_at, rdict) = (ds~load_uint(32), ds~load_uint(256), ds~load_uint(32), ds~load_dict()); ds.end_parse(); throw_unless(33, msg_seqno == stored_seqno); ifnot (msg_seqno) { @@ -24,14 +27,15 @@ _ days_passed() inline { set_data(begin_cell() .store_uint(stored_seqno + 1, 32) .store_uint(public_key, 256) + .store_uint(start_at, 32) .store_dict(rdict) .end_cell()); return (); } throw_unless(34, check_signature(slice_hash(in_msg), signature, public_key)); accept_message(); - var ts = days_passed(); - var (_, value, found) = rdict.idict_get_preveq?(16, ts); + var ts = seconds_passed(start_at, now()); + var (_, value, found) = rdict.idict_get_preveq?(32, ts); if (found) { raw_reserve(value~load_grams(), 2); } @@ -45,6 +49,7 @@ _ days_passed() inline { set_data(begin_cell() .store_uint(stored_seqno + 1, 32) .store_uint(public_key, 256) + .store_uint(start_at, 32) .store_dict(rdict) .end_cell()); } @@ -61,15 +66,23 @@ int get_public_key() method_id { return cs.preload_uint(256); } -int balance() method_id { +int compute_balance_at(int utime) inline_ref { var ds = get_data().begin_parse().skip_bits(32 + 256); - var rdict = ds~load_dict(); + var (start_at, rdict) = (ds~load_uint(32), ds~load_dict()); ds.end_parse(); - var ts = days_passed(); + var ts = seconds_passed(start_at, utime); var balance = get_balance().pair_first(); - var (_, value, found) = rdict.idict_get_preveq?(16, ts); + var (_, value, found) = rdict.idict_get_preveq?(32, ts); if (found) { balance = max(balance - value~load_grams(), 0); } return balance; } + +int balance_at(int utime) method_id { + return compute_balance_at(utime); +} + +int balance() method_id { + return compute_balance_at(now()); +} diff --git a/crypto/smartcont/restricted-wallet3-code.fc b/crypto/smartcont/restricted-wallet3-code.fc new file mode 100644 index 00000000..aab86065 --- /dev/null +++ b/crypto/smartcont/restricted-wallet3-code.fc @@ -0,0 +1,103 @@ +;; Restricted wallet initialized by a third party (a variant of restricted-wallet2-code.fc) +;; restricts access to parts of balance until certain dates + +() recv_internal(slice in_msg) impure { + ;; do nothing for internal messages +} + +_ seconds_passed(int start_at, int utime) inline_ref { + ifnot (start_at) { + var p = config_param(-13); + start_at = null?(p) ? 0 : begin_parse(p).preload_uint(32); + } + return start_at ? utime - start_at : -1; +} + +() recv_external(slice in_msg) impure { + var signature = in_msg~load_bits(512); + var cs = in_msg; + var (subwallet_id, valid_until, msg_seqno) = (cs~load_uint(32), cs~load_uint(32), cs~load_uint(32)); + throw_if(35, valid_until <= now()); + var ds = get_data().begin_parse(); + var (stored_seqno, stored_subwallet, public_key) = (ds~load_uint(32), ds~load_uint(32), ds~load_uint(256)); + throw_unless(33, msg_seqno == stored_seqno); + throw_unless(34, subwallet_id == stored_subwallet); + throw_unless(36, check_signature(slice_hash(in_msg), signature, public_key)); + ifnot (msg_seqno) { + public_key = ds~load_uint(256); ;; load "final" public key + ds.end_parse(); + cs~touch(); + var (start_at, rdict) = (cs~load_uint(32), cs~load_dict()); + cs.end_parse(); + accept_message(); + set_data(begin_cell() + .store_uint(stored_seqno + 1, 32) + .store_uint(stored_subwallet, 32) + .store_uint(public_key, 256) + .store_uint(start_at, 32) + .store_dict(rdict) + .end_cell()); + return (); + } + var (start_at, rdict) = (ds~load_uint(32), ds~load_dict()); + ds.end_parse(); + accept_message(); + var ts = seconds_passed(start_at, now()); + var (_, value, found) = rdict.idict_get_preveq?(32, ts); + if (found) { + raw_reserve(value~load_grams(), 2); + } + cs~touch(); + while (cs.slice_refs()) { + var mode = cs~load_uint(8); + var msg = cs~load_ref(); + send_raw_message(msg, mode); + } + cs.end_parse(); + set_data(begin_cell() + .store_uint(stored_seqno + 1, 32) + .store_uint(stored_subwallet, 32) + .store_uint(public_key, 256) + .store_uint(start_at, 32) + .store_dict(rdict) + .end_cell()); +} + +;; Get methods + +int seqno() method_id { + return get_data().begin_parse().preload_uint(32); +} + +int wallet_id() method_id { + var ds = get_data().begin_parse(); + ds~load_uint(32); + return ds.preload_uint(32); +} + +int get_public_key() method_id { + var ds = get_data().begin_parse(); + ds~load_uint(32 + 32); + return ds.preload_uint(256); +} + +int compute_balance_at(int utime) inline_ref { + var ds = get_data().begin_parse().skip_bits(32 + 32 + 256); + var (start_at, rdict) = (ds~load_uint(32), ds~load_dict()); + ds.end_parse(); + var ts = seconds_passed(start_at, utime); + var balance = get_balance().pair_first(); + var (_, value, found) = rdict.idict_get_preveq?(32, ts); + if (found) { + balance = max(balance - value~load_grams(), 0); + } + return balance; +} + +int balance_at(int utime) method_id { + return compute_balance_at(utime); +} + +int balance() method_id { + return compute_balance_at(now()); +} diff --git a/crypto/smartcont/show-addr.fif b/crypto/smartcont/show-addr.fif index 86b31463..c421a879 100755 --- a/crypto/smartcont/show-addr.fif +++ b/crypto/smartcont/show-addr.fif @@ -18,5 +18,7 @@ dup Blen { 32 B>i@ } { drop Basechain } cond constant wallet_wc file-base +".pk" dup file-exists? { dup file>B dup Blen 32 <> abort"Private key must be exactly 32 bytes long" - =: wallet_pk ."Private key available in file " type cr + tuck =: wallet_pk ."Private key available in file " type cr + priv>pub 256 B>u@ + dup ."Corresponding public key is " .pubkey ." = " 64X. cr } { ."Private key file " type ." not found" cr } cond diff --git a/crypto/smc-envelope/GenericAccount.cpp b/crypto/smc-envelope/GenericAccount.cpp index 202ff2dc..91b2edf7 100644 --- a/crypto/smc-envelope/GenericAccount.cpp +++ b/crypto/smc-envelope/GenericAccount.cpp @@ -21,6 +21,31 @@ #include "block/block-auto.h" #include "block/block-parse.h" namespace ton { + +namespace smc { +td::Ref pack_grams(td::uint64 amount) { + vm::CellBuilder cb; + block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(amount)); + return vm::load_cell_slice_ref(cb.finalize()); +} + +bool unpack_grams(td::Ref cs, td::uint64& amount) { + td::RefInt256 got; + if (!block::tlb::t_Grams.as_integer_to(cs, got)) { + return false; + } + if (!got->unsigned_fits_bits(63)) { + return false; + } + auto x = got->to_long(); + if (x < 0) { + return false; + } + amount = x; + return true; +} +} // namespace smc + td::Ref GenericAccount::get_init_state(td::Ref code, td::Ref data) noexcept { return vm::CellBuilder() .store_zeroes(2) @@ -47,7 +72,7 @@ void GenericAccount::store_int_message(vm::CellBuilder& cb, const block::StdAddr .store_long(dest_address.workchain, 8) .store_int256(dest_addr, 256); block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(gramms)); - cb.store_zeroes(9 + 64 + 32 + 1 + 1); + cb.store_zeroes(9 + 64 + 32); } td::Ref GenericAccount::create_ext_message(const block::StdAddress& address, td::Ref new_state, diff --git a/crypto/smc-envelope/GenericAccount.h b/crypto/smc-envelope/GenericAccount.h index 27e43e25..6100b6dd 100644 --- a/crypto/smc-envelope/GenericAccount.h +++ b/crypto/smc-envelope/GenericAccount.h @@ -23,6 +23,10 @@ #include "SmartContract.h" namespace ton { +namespace smc { +td::Ref pack_grams(td::uint64 amount); +bool unpack_grams(td::Ref cs, td::uint64& amount); +} // namespace smc class GenericAccount { public: static td::Ref get_init_state(td::Ref code, td::Ref data) noexcept; diff --git a/crypto/smc-envelope/HighloadWallet.cpp b/crypto/smc-envelope/HighloadWallet.cpp index af529a1b..7996d095 100644 --- a/crypto/smc-envelope/HighloadWallet.cpp +++ b/crypto/smc-envelope/HighloadWallet.cpp @@ -73,17 +73,11 @@ td::Ref HighloadWallet::make_a_gift_message(const td::Ed25519::Private for (size_t i = 0; i < gifts.size(); i++) { auto& gift = gifts[i]; td::int32 send_mode = 3; - auto gramms = gift.gramms; - if (gramms == -1) { - gramms = 0; + if (gift.gramms == -1) { send_mode += 128; } + auto message_inner = create_int_message(gift); vm::CellBuilder cb; - GenericAccount::store_int_message(cb, gift.destination, gramms); - cb.store_bytes("\0\0\0\0", 4); - vm::CellString::store(cb, gift.message, 35 * 8).ensure(); - auto message_inner = cb.finalize(); - cb = {}; cb.store_long(send_mode, 8).store_ref(message_inner); auto key = messages.integer_key(td::make_refint(i), 16, false); messages.set_builder(key.bits(), 16, cb); diff --git a/crypto/smc-envelope/HighloadWalletV2.cpp b/crypto/smc-envelope/HighloadWalletV2.cpp index dbdb5234..bae249b4 100644 --- a/crypto/smc-envelope/HighloadWalletV2.cpp +++ b/crypto/smc-envelope/HighloadWalletV2.cpp @@ -73,18 +73,11 @@ td::Ref HighloadWalletV2::make_a_gift_message(const td::Ed25519::Priva for (size_t i = 0; i < gifts.size(); i++) { auto& gift = gifts[i]; td::int32 send_mode = 3; - auto gramms = gift.gramms; - if (gramms == -1) { - gramms = 0; + if (gift.gramms == -1) { send_mode += 128; } vm::CellBuilder cb; - GenericAccount::store_int_message(cb, gift.destination, gramms); - cb.store_bytes("\0\0\0\0", 4); - vm::CellString::store(cb, gift.message, 35 * 8).ensure(); - auto message_inner = cb.finalize(); - cb = {}; - cb.store_long(send_mode, 8).store_ref(message_inner); + cb.store_long(send_mode, 8).store_ref(create_int_message(gift)); auto key = messages.integer_key(td::make_refint(i), 16, false); messages.set_builder(key.bits(), 16, cb); } diff --git a/crypto/smc-envelope/PaymentChannel.cpp b/crypto/smc-envelope/PaymentChannel.cpp new file mode 100644 index 00000000..a7367922 --- /dev/null +++ b/crypto/smc-envelope/PaymentChannel.cpp @@ -0,0 +1,264 @@ +#include "PaymentChannel.h" +#include "GenericAccount.h" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "Ed25519.h" +#include "block/block-auto.h" +#include "block/block-parse.h" + +#include "SmartContract.h" +#include "SmartContractCode.h" + +namespace ton { +using smc::pack_grams; +using smc::unpack_grams; +namespace pchan { + +td::Ref Config::serialize() const { + block::gen::ChanConfig::Record rec; + + vm::CellBuilder a_addr_cb; + block::tlb::t_MsgAddressInt.store_std_address(a_addr_cb, a_addr); + rec.a_addr = a_addr_cb.finalize_novm(); + + vm::CellBuilder b_addr_cb; + block::tlb::t_MsgAddressInt.store_std_address(b_addr_cb, b_addr); + rec.b_addr = b_addr_cb.finalize_novm(); + + rec.a_key.as_slice().copy_from(a_key); + rec.b_key.as_slice().copy_from(b_key); + rec.init_timeout = init_timeout; + rec.close_timeout = close_timeout; + rec.channel_id = channel_id; + + td::Ref res; + CHECK(tlb::pack_cell(res, rec)); + return res; +} + +td::Ref MsgInit::serialize() const { + block::gen::ChanMsg::Record_chan_msg_init rec; + rec.min_A = pack_grams(min_A); + rec.min_B = pack_grams(min_B); + rec.inc_A = pack_grams(inc_A); + rec.inc_B = pack_grams(inc_B); + rec.channel_id = channel_id; + + td::Ref res; + CHECK(tlb::pack_cell(res, rec)); + return res; +} + +td::Ref Promise::serialize() const { + block::gen::ChanPromise::Record rec; + rec.channel_id = channel_id; + rec.promise_A = pack_grams(promise_A); + rec.promise_B = pack_grams(promise_B); + td::Ref res; + CHECK(tlb::pack_cell(res, rec)); + return res; +} + +td::SecureString sign(const td::Ref& msg, const td::Ed25519::PrivateKey* key) { + return key->sign(msg->get_hash().as_slice()).move_as_ok(); +} + +td::Ref maybe_sign(const td::Ref& msg, const td::Ed25519::PrivateKey* key) { + if (!key) { + return {}; + } + return vm::CellBuilder().store_bytes(sign(msg, key).as_slice()).finalize(); +} + +td::Ref maybe_ref(td::Ref msg) { + vm::CellBuilder cb; + CHECK(cb.store_maybe_ref(msg)); + return vm::load_cell_slice_ref(cb.finalize()); +} + +td::Ref MsgClose::serialize() const { + block::gen::ChanMsg::Record_chan_msg_close rec; + rec.extra_A = pack_grams(extra_A); + rec.extra_B = pack_grams(extra_B); + rec.promise = signed_promise; + + td::Ref res; + CHECK(tlb::pack_cell(res, rec)); + return res; +} + +td::Ref MsgTimeout::serialize() const { + block::gen::ChanMsg::Record_chan_msg_timeout rec; + td::Ref res; + CHECK(tlb::pack_cell(res, rec)); + return res; +} + +td::SecureString SignedPromise::signature(const td::Ed25519::PrivateKey* key, const td::Ref& promise) { + return sign(promise, key); +} +td::Ref SignedPromise::create_and_serialize(td::Slice signature, const td::Ref& promise) { + block::gen::ChanSignedPromise::Record rec; + rec.promise = vm::load_cell_slice_ref(promise); + LOG(ERROR) << "signature.size() = " << signature.size(); + rec.sig = maybe_ref(vm::CellBuilder().store_bytes(signature).finalize()); + td::Ref res; + CHECK(tlb::pack_cell(res, rec)); + return res; +} +td::Ref SignedPromise::create_and_serialize(const td::Ed25519::PrivateKey* key, + const td::Ref& promise) { + block::gen::ChanSignedPromise::Record rec; + rec.promise = vm::load_cell_slice_ref(promise); + rec.sig = maybe_ref(maybe_sign(promise, key)); + td::Ref res; + CHECK(tlb::pack_cell(res, rec)); + return res; +} + +bool SignedPromise::unpack(td::Ref cell) { + block::gen::ChanSignedPromise::Record rec; + if (!tlb::unpack_cell(cell, rec)) { + return false; + } + block::gen::ChanPromise::Record rec_promise; + if (!tlb::csr_unpack(rec.promise, rec_promise)) { + return false; + } + promise.channel_id = rec_promise.channel_id; + if (!unpack_grams(rec_promise.promise_A, promise.promise_A)) { + return false; + } + if (!unpack_grams(rec_promise.promise_B, promise.promise_B)) { + return false; + } + td::Ref sig_cell; + if (!rec.sig->prefetch_maybe_ref(sig_cell)) { + return false; + } + td::SecureString signature(64); + vm::CellSlice cs = vm::load_cell_slice(sig_cell); + if (!cs.prefetch_bytes(signature.as_mutable_slice())) { + return false; + } + o_signature = std::move(signature); + return true; +} + +td::Ref StateInit::serialize() const { + block::gen::ChanState::Record_chan_state_init rec; + rec.expire_at = expire_at; + rec.min_A = pack_grams(min_A); + rec.min_B = pack_grams(min_B); + rec.A = pack_grams(A); + rec.B = pack_grams(B); + rec.signed_A = signed_A; + rec.signed_B = signed_B; + td::Ref res; + CHECK(tlb::pack_cell(res, rec)); + return res; +} + +td::Ref Data::serialize() const { + block::gen::ChanData::Record rec; + rec.config = config; + rec.state = state; + td::Ref res; + CHECK(block::gen::t_ChanData.cell_pack(res, rec)); + return res; +} + +td::Ref Data::init_state() { + return StateInit().serialize(); +} +} // namespace pchan + +td::Result PaymentChannel::get_info() const { + block::gen::ChanData::Record data_rec; + if (!tlb::unpack_cell(get_state().data, data_rec)) { + return td::Status::Error("Can't unpack data"); + } + block::gen::ChanConfig::Record config_rec; + if (!tlb::unpack_cell(data_rec.config, config_rec)) { + return td::Status::Error("Can't unpack config"); + } + pchan::Config config; + config.a_key = td::SecureString(config_rec.a_key.as_slice()); + config.b_key = td::SecureString(config_rec.b_key.as_slice()); + block::tlb::t_MsgAddressInt.extract_std_address(vm::load_cell_slice_ref(config_rec.a_addr), config.a_addr); + block::tlb::t_MsgAddressInt.extract_std_address(vm::load_cell_slice_ref(config_rec.b_addr), config.b_addr); + config.init_timeout = static_cast(config_rec.init_timeout); + config.close_timeout = static_cast(config_rec.close_timeout); + config.channel_id = static_cast(config_rec.channel_id); + + auto state_cs = vm::load_cell_slice(data_rec.state); + Info res; + switch (block::gen::t_ChanState.check_tag(state_cs)) { + case block::gen::ChanState::chan_state_init: { + pchan::StateInit state; + block::gen::ChanState::Record_chan_state_init state_rec; + if (!tlb::unpack_cell(data_rec.state, state_rec)) { + return td::Status::Error("Can't unpack state"); + } + bool ok = unpack_grams(state_rec.A, state.A) && unpack_grams(state_rec.B, state.B) && + unpack_grams(state_rec.min_A, state.min_A) && unpack_grams(state_rec.min_B, state.min_B); + state.expire_at = state_rec.expire_at; + state.signed_A = state_rec.signed_A; + state.signed_B = state_rec.signed_B; + if (!ok) { + return td::Status::Error("Can't unpack state"); + } + res.state = std::move(state); + break; + } + case block::gen::ChanState::chan_state_close: { + pchan::StateClose state; + block::gen::ChanState::Record_chan_state_close state_rec; + if (!tlb::unpack_cell(data_rec.state, state_rec)) { + return td::Status::Error("Can't unpack state"); + } + bool ok = unpack_grams(state_rec.A, state.A) && unpack_grams(state_rec.B, state.B) && + unpack_grams(state_rec.promise_A, state.promise_A) && + unpack_grams(state_rec.promise_B, state.promise_B); + state.expire_at = state_rec.expire_at; + state.signed_A = state_rec.signed_A; + state.signed_B = state_rec.signed_B; + if (!ok) { + return td::Status::Error("Can't unpack state"); + } + res.state = std::move(state); + break; + } + case block::gen::ChanState::chan_state_payout: { + pchan::StatePayout state; + block::gen::ChanState::Record_chan_state_payout state_rec; + if (!tlb::unpack_cell(data_rec.state, state_rec)) { + return td::Status::Error("Can't unpack state"); + } + bool ok = unpack_grams(state_rec.A, state.A) && unpack_grams(state_rec.B, state.B); + if (!ok) { + return td::Status::Error("Can't unpack state"); + } + res.state = std::move(state); + break; + } + default: + return td::Status::Error("Can't unpack state"); + } + + res.config = std::move(config); + res.description = block::gen::t_ChanState.as_string_ref(data_rec.state); + + return std::move(res); +} // namespace ton + +td::optional PaymentChannel::guess_revision(const vm::Cell::Hash& code_hash) { + for (auto i : ton::SmartContractCode::get_revisions(ton::SmartContractCode::PaymentChannel)) { + auto code = SmartContractCode::get_code(SmartContractCode::PaymentChannel, i); + if (code->get_hash() == code_hash) { + return i; + } + } + return {}; +} +} // namespace ton diff --git a/crypto/smc-envelope/PaymentChannel.h b/crypto/smc-envelope/PaymentChannel.h new file mode 100644 index 00000000..f050f330 --- /dev/null +++ b/crypto/smc-envelope/PaymentChannel.h @@ -0,0 +1,251 @@ +#pragma once +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "Ed25519.h" +#include "block/block-auto.h" +#include "block/block-parse.h" + +#include "td/utils/Variant.h" + +#include "SmartContract.h" +#include "SmartContractCode.h" + +namespace ton { +namespace pchan { + +// +// Payment channels +// +struct Config { + td::uint32 init_timeout{0}; + td::uint32 close_timeout{0}; + td::SecureString a_key; + td::SecureString b_key; + block::StdAddress a_addr; + block::StdAddress b_addr; + td::uint64 channel_id{0}; + + td::Ref serialize() const; +}; + +struct MsgInit { + td::uint64 inc_A{0}; + td::uint64 inc_B{0}; + td::uint64 min_A{0}; + td::uint64 min_B{0}; + td::uint64 channel_id{0}; + + td::Ref serialize() const; +}; + +struct Promise { + td::uint64 channel_id; + td::uint64 promise_A{0}; + td::uint64 promise_B{0}; + td::Ref serialize() const; +}; + +td::Ref maybe_sign(const td::Ref& msg, const td::Ed25519::PrivateKey* key); +td::Ref maybe_ref(td::Ref msg); + +struct MsgClose { + td::uint64 extra_A{0}; + td::uint64 extra_B{0}; + td::Ref signed_promise; + td::Ref serialize() const; +}; + +struct MsgTimeout { + td::Ref serialize() const; +}; + +struct SignedPromise { + Promise promise; + td::optional o_signature; + + bool unpack(td::Ref cell); + static td::SecureString signature(const td::Ed25519::PrivateKey* key, const td::Ref& promise); + static td::Ref create_and_serialize(td::Slice signature, const td::Ref& promise); + static td::Ref create_and_serialize(const td::Ed25519::PrivateKey* key, const td::Ref& promise); +}; + +struct StateInit { + bool signed_A{false}; + bool signed_B{false}; + td::uint64 min_A{0}; + td::uint64 min_B{0}; + td::uint64 A{0}; + td::uint64 B{0}; + td::uint32 expire_at{0}; + + td::Ref serialize() const; +}; + +struct StateClose { + bool signed_A{false}; + bool signed_B{false}; + td::uint64 promise_A{0}; + td::uint64 promise_B{0}; + td::uint64 A{0}; + td::uint64 B{0}; + td::uint32 expire_at{0}; +}; + +struct StatePayout { + td::uint64 A{0}; + td::uint64 B{0}; +}; + +struct Data { + td::Ref config; + td::Ref state; + + static td::Ref init_state(); + + td::Ref serialize() const; +}; + +template +struct MsgBuilder { + td::Ed25519::PrivateKey* a_key{nullptr}; + td::Ed25519::PrivateKey* b_key{nullptr}; + + T&& with_a_key(td::Ed25519::PrivateKey* key) && { + a_key = key; + return static_cast(*this); + } + T&& with_b_key(td::Ed25519::PrivateKey* key) && { + b_key = key; + return static_cast(*this); + } + + td::Ref finalize() && { + block::gen::ChanSignedMsg::Record rec; + auto msg = static_cast(*this).msg.serialize(); + rec.msg = vm::load_cell_slice_ref(msg); + rec.sig_A = maybe_ref(maybe_sign(msg, a_key)); + rec.sig_B = maybe_ref(maybe_sign(msg, b_key)); + td::Ref res; + CHECK(tlb::pack_cell(res, rec)); + return res; + } +}; + +struct MsgInitBuilder : public MsgBuilder { + MsgInit msg; + + MsgInitBuilder&& min_A(td::uint64 value) && { + msg.min_A = value; + return std::move(*this); + } + MsgInitBuilder&& min_B(td::uint64 value) && { + msg.min_B = value; + return std::move(*this); + } + MsgInitBuilder&& inc_A(td::uint64 value) && { + msg.inc_A = value; + return std::move(*this); + } + MsgInitBuilder&& inc_B(td::uint64 value) && { + msg.inc_B = value; + return std::move(*this); + } + MsgInitBuilder&& channel_id(td::uint64 value) && { + msg.channel_id = value; + return std::move(*this); + } +}; + +struct MsgTimeoutBuilder : public MsgBuilder { + MsgTimeout msg; +}; + +struct MsgCloseBuilder : public MsgBuilder { + MsgClose msg; + + MsgCloseBuilder&& extra_A(td::uint64 value) && { + msg.extra_A = value; + return std::move(*this); + } + MsgCloseBuilder&& extra_B(td::uint64 value) && { + msg.extra_B = value; + return std::move(*this); + } + MsgCloseBuilder&& signed_promise(td::Ref signed_promise) && { + msg.signed_promise = vm::load_cell_slice_ref(signed_promise); + return std::move(*this); + } +}; + +struct SignedPromiseBuilder { + Promise promise; + td::optional o_signature; + td::Ed25519::PrivateKey* key{nullptr}; + + SignedPromiseBuilder& with_key(td::Ed25519::PrivateKey* key) { + this->key = key; + return *this; + } + SignedPromiseBuilder& promise_A(td::uint64 value) { + promise.promise_A = value; + return *this; + } + SignedPromiseBuilder& promise_B(td::uint64 value) { + promise.promise_B = value; + return *this; + } + SignedPromiseBuilder& channel_id(td::uint64 value) { + promise.channel_id = value; + return *this; + } + SignedPromiseBuilder& signature(td::SecureString signature) { + o_signature = std::move(signature); + return *this; + } + + bool check_signature(td::Slice signature, const td::Ed25519::PublicKey& pk) { + return pk.verify_signature(promise.serialize()->get_hash().as_slice(), signature).is_ok(); + } + td::SecureString calc_signature() { + CHECK(key); + return SignedPromise::signature(key, promise.serialize()); + } + td::Ref finalize() { + if (o_signature) { + return SignedPromise::create_and_serialize(o_signature.value().copy(), promise.serialize()); + } else { + return SignedPromise::create_and_serialize(key, promise.serialize()); + } + } +}; + +} // namespace pchan + +class PaymentChannel : public SmartContract { + public: + PaymentChannel(State state) : SmartContract(std::move(state)) { + } + + struct Info { + pchan::Config config; + td::Variant state; + std::string description; + }; + td::Result get_info() const; + + static td::Ref create(State state) { + return td::Ref(true, std::move(state)); + } + static td::optional guess_revision(const vm::Cell::Hash& code_hash); + static td::Ref create(const pchan::Config& config, td::int32 revision) { + State state; + state.code = SmartContractCode::get_code(SmartContractCode::PaymentChannel, revision); + pchan::Data data; + data.config = config.serialize(); + pchan::StateInit init; + data.state = init.serialize(); + state.data = data.serialize(); + return create(std::move(state)); + } +}; +} // namespace ton diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 42a6ea8a..79a2a823 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -32,19 +32,19 @@ namespace ton { namespace { -td::Ref prepare_vm_stack(td::Ref body) { +td::Ref prepare_vm_stack(td::RefInt256 amount, td::Ref body) { td::Ref stack_ref{true}; td::RefInt256 acc_addr{true}; //CHECK(acc_addr.write().import_bits(account.addr.cbits(), 256)); vm::Stack& stack = stack_ref.write(); stack.push_int(td::make_refint(10000000000)); - stack.push_int(td::make_refint(10000000000)); + stack.push_int(std::move(amount)); stack.push_cell(vm::CellBuilder().finalize()); stack.push_cellslice(std::move(body)); return stack_ref; } -td::Ref prepare_vm_c7(td::uint32 now) { +td::Ref prepare_vm_c7(td::uint32 now, td::uint64 balance) { // TODO: fix initialization of c7 td::BitArray<256> rand_seed; rand_seed.as_slice().fill(0); @@ -58,7 +58,7 @@ td::Ref prepare_vm_c7(td::uint32 now) { td::make_refint(0), // block_lt:Integer td::make_refint(0), // trans_lt:Integer std::move(rand_seed_int), // rand_seed:Integer - block::CurrencyCollection(1000000000).as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] + block::CurrencyCollection(balance).as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] vm::load_cell_slice_ref(vm::CellBuilder().finalize()) // myself:MsgAddressInt //vm::StackEntry::maybe(td::Ref()) ); // global_config:(Maybe Cell) ] = SmartContractInfo; @@ -66,6 +66,15 @@ td::Ref prepare_vm_c7(td::uint32 now) { return vm::make_tuple_ref(std::move(tuple)); } +static int output_actions_count(td::Ref list) { + int i = -1; + do { + ++i; + list = load_cell_slice(std::move(list)).prefetch_ref(); + } while (list.not_null()); + return i; +} + SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref stack, td::Ref c7, vm::GasLimits gas, bool ignore_chksig) { auto gas_credit = gas.gas_credit; @@ -123,6 +132,8 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref cell, Args args) { - return run_method(args.set_stack(prepare_vm_stack(vm::load_cell_slice_ref(cell))).set_method_id(-1)); + return run_method( + args.set_stack(prepare_vm_stack(td::make_refint(0), vm::load_cell_slice_ref(cell))).set_method_id(-1)); +} +SmartContract::Answer SmartContract::send_internal_message(td::Ref cell, Args args) { + return run_method( + args.set_stack(prepare_vm_stack(td::make_refint(args.amount), vm::load_cell_slice_ref(cell))).set_method_id(0)); } } // namespace ton diff --git a/crypto/smc-envelope/SmartContract.h b/crypto/smc-envelope/SmartContract.h index 5bc70a4f..89f5c8c7 100644 --- a/crypto/smc-envelope/SmartContract.h +++ b/crypto/smc-envelope/SmartContract.h @@ -57,6 +57,8 @@ class SmartContract : public td::CntObject { td::optional> stack; td::optional now; bool ignore_chksig{false}; + td::uint64 amount{0}; + td::uint64 balance{0}; Args() { } @@ -95,6 +97,14 @@ class SmartContract : public td::CntObject { this->ignore_chksig = ignore_chksig; return std::move(*this); } + Args&& set_amount(td::uint64 amount) { + this->amount = amount; + return std::move(*this); + } + Args&& set_balance(td::uint64 balance) { + this->balance = balance; + return std::move(*this); + } td::Result get_method_id() const { if (!method_id) { @@ -109,6 +119,7 @@ class SmartContract : public td::CntObject { Answer run_get_method(Args args = {}) const; Answer run_get_method(td::Slice method, Args args = {}) const; Answer send_external_message(td::Ref cell, Args args = {}); + Answer send_internal_message(td::Ref cell, Args args = {}); size_t code_size() const; size_t data_size() const; @@ -122,6 +133,9 @@ class SmartContract : public td::CntObject { const State& get_state() const { return state_; } + CntObject* make_copy() const override { + return new SmartContract(state_); + } protected: State state_; diff --git a/crypto/smc-envelope/SmartContractCode.cpp b/crypto/smc-envelope/SmartContractCode.cpp index 27bf58ab..99ed8384 100644 --- a/crypto/smc-envelope/SmartContractCode.cpp +++ b/crypto/smc-envelope/SmartContractCode.cpp @@ -44,6 +44,8 @@ const auto& get_map() { #include "smartcont/auto/highload-wallet-code.cpp" #include "smartcont/auto/highload-wallet-v2-code.cpp" #include "smartcont/auto/dns-manual-code.cpp" +#include "smartcont/auto/payment-channel-code.cpp" +#include "smartcont/auto/restricted-wallet3-code.cpp" with_tvm_code("highload-wallet-r1", "te6ccgEBBgEAhgABFP8A9KQT9KDyyAsBAgEgAgMCAUgEBQC88oMI1xgg0x/TH9Mf+CMTu/Jj7UTQ0x/TH9P/" @@ -96,6 +98,14 @@ const auto& get_map() { "FwCEMQLTAAHAAZPUAdCY0wUBqgLXGAHiINdJwg/" "ypiB41yLXCwfyaHBTEddJqTYCmNMHAcAAEqEB5DDIywYBzxbJ0FADACBZ9KhvpSCUAvQEMJIybeICACg0A4AQ9FqZECOECUBE8AEBkjAx4gBmM" "SLAFZwy9AQQI4QJUELwAQHgIsAWmDIChAn0czAB4DAyIMAfkzD0BODAIJJtAeDyLG0B"); + with_tvm_code( + "restricted-wallet3-r1", + "te6ccgECEgEAAUsAART/APSkE/S88sgLAQIBIAIDAgFIBAUD+PKDCNcYINMf0x/THwL4I7vyY+1E0NMf0x/T/" + "1NDuvKhUWK68qIG+QFUEHb5EPKkAY4fMwHT/9EB0x/0BNH4AAOkyMsfFMsfy/8Syx/0AMntVOEC0x/" + "0BNH4ACH4I9s8IYAg9HtvpTGW+gAwcvsCkTDiApMg10qK6NECpMgPEBEABNAwAgEgBgcCASAICQIBSAwNAgFuCgsAEbjJftRNDXCx+" + "AAXrc52omhpn5jrhf/AABesePaiaGmPmOuFj8ABDbbYHwR7Z5AOAQm1B1tnkA4BTu1E0IEBQNch0x/" + "0BNEC2zz4J28QAoAg9HtvpTGX+gAwoXC2CZEw4g8AOiGOETGA8/gzIG6SMHCU0NcLH+IB3yGSAaGSW3/iAAzTB9QC+wAAHssfFMsfEsv/yx/" + "0AMntVA=="); return map; }(); return map; @@ -103,7 +113,6 @@ const auto& get_map() { } // namespace td::Result> SmartContractCode::load(td::Slice name) { - LOG(ERROR) << "LOAD " << name; auto& map = get_map(); auto it = map.find(name); if (it == map.end()) { @@ -146,6 +155,14 @@ td::Span SmartContractCode::get_revisions(Type type) { static int res[] = {-1, 1}; return res; } + case Type::PaymentChannel: { + static int res[] = {-1}; + return res; + } + case Type::RestrictedWallet: { + static int res[] = {-1, 1}; + return res; + } } UNREACHABLE(); return {}; @@ -190,6 +207,10 @@ td::Ref SmartContractCode::get_code(Type type, int ext_revision) { return "multisig"; case Type::ManualDns: return "dns-manual"; + case Type::PaymentChannel: + return "payment-channel"; + case Type::RestrictedWallet: + return "restricted-wallet3"; } UNREACHABLE(); return ""; diff --git a/crypto/smc-envelope/SmartContractCode.h b/crypto/smc-envelope/SmartContractCode.h index 1ea2274c..e9ca1c54 100644 --- a/crypto/smc-envelope/SmartContractCode.h +++ b/crypto/smc-envelope/SmartContractCode.h @@ -16,6 +16,7 @@ Copyright 2017-2020 Telegram Systems LLP */ +#pragma once #include "vm/cells.h" #include "td/utils/Span.h" @@ -25,7 +26,18 @@ class SmartContractCode { public: static td::Result> load(td::Slice name); - enum Type { WalletV1 = 1, WalletV1Ext, WalletV2, WalletV3, HighloadWalletV1, HighloadWalletV2, ManualDns, Multisig }; + enum Type { + WalletV1 = 1, + WalletV1Ext, + WalletV2, + WalletV3, + HighloadWalletV1, + HighloadWalletV2, + ManualDns, + Multisig, + PaymentChannel, + RestrictedWallet + }; static td::Span get_revisions(Type type); static td::Result validate_revision(Type type, int revision); static td::Ref get_code(Type type, int revision = 0); diff --git a/crypto/smc-envelope/TestGiver.cpp b/crypto/smc-envelope/TestGiver.cpp index f9ed60bf..2a169183 100644 --- a/crypto/smc-envelope/TestGiver.cpp +++ b/crypto/smc-envelope/TestGiver.cpp @@ -43,12 +43,7 @@ td::Ref TestGiver::make_a_gift_message_static(td::uint32 seqno, td::Sp for (auto& gift : gifts) { td::int32 send_mode = 1; - auto gramms = gift.gramms; - vm::CellBuilder cbi; - GenericAccount::store_int_message(cbi, gift.destination, gramms); - store_gift_message(cbi, gift); - auto message_inner = cbi.finalize(); - cb.store_long(send_mode, 8).store_ref(std::move(message_inner)); + cb.store_long(send_mode, 8).store_ref(create_int_message(gift)); } return cb.finalize(); diff --git a/crypto/smc-envelope/TestWallet.cpp b/crypto/smc-envelope/TestWallet.cpp index 4f4dcd8d..f218cf7c 100644 --- a/crypto/smc-envelope/TestWallet.cpp +++ b/crypto/smc-envelope/TestWallet.cpp @@ -46,18 +46,11 @@ td::Ref TestWallet::make_a_gift_message_static(const td::Ed25519::Priv for (auto& gift : gifts) { td::int32 send_mode = 3; - auto gramms = gift.gramms; - if (gramms == -1) { - gramms = 0; + if (gift.gramms == -1) { send_mode += 128; } - vm::CellBuilder cbi; - GenericAccount::store_int_message(cbi, gift.destination, gramms); - store_gift_message(cbi, gift); - auto message_inner = cbi.finalize(); - cb.store_long(send_mode, 8).store_ref(std::move(message_inner)); + cb.store_long(send_mode, 8).store_ref(create_int_message(gift)); } - auto message_outer = cb.finalize(); auto signature = private_key.sign(message_outer->get_hash().as_slice()).move_as_ok(); return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); diff --git a/crypto/smc-envelope/Wallet.cpp b/crypto/smc-envelope/Wallet.cpp index ef637fb9..a09ddff9 100644 --- a/crypto/smc-envelope/Wallet.cpp +++ b/crypto/smc-envelope/Wallet.cpp @@ -52,16 +52,10 @@ td::Ref Wallet::make_a_gift_message(const td::Ed25519::PrivateKey& pri for (auto& gift : gifts) { td::int32 send_mode = 3; - auto gramms = gift.gramms; - if (gramms == -1) { - gramms = 0; + if (gift.gramms == -1) { send_mode += 128; } - vm::CellBuilder cbi; - GenericAccount::store_int_message(cbi, gift.destination, gramms); - store_gift_message(cbi, gift); - auto message_inner = cbi.finalize(); - cb.store_long(send_mode, 8).store_ref(std::move(message_inner)); + cb.store_long(send_mode, 8).store_ref(create_int_message(gift)); } auto message_outer = cb.finalize(); diff --git a/crypto/smc-envelope/WalletInterface.h b/crypto/smc-envelope/WalletInterface.h index c8666f53..d4113a6f 100644 --- a/crypto/smc-envelope/WalletInterface.h +++ b/crypto/smc-envelope/WalletInterface.h @@ -24,6 +24,7 @@ #include "vm/cells/CellString.h" #include "SmartContract.h" +#include "GenericAccount.h" namespace ton { class WalletInterface { @@ -36,6 +37,7 @@ class WalletInterface { std::string message; td::Ref body; + td::Ref init_state; }; virtual ~WalletInterface() { @@ -48,15 +50,29 @@ class WalletInterface { return td::Status::Error("Unsupported"); } - td::Result> get_init_message(const td::Ed25519::PrivateKey &private_key, - td::uint32 valid_until = std::numeric_limits::max()) { + td::Result> get_init_message( + const td::Ed25519::PrivateKey &private_key, + td::uint32 valid_until = std::numeric_limits::max()) const { return make_a_gift_message(private_key, valid_until, {}); } + static td::Ref create_int_message(const Gift &gift) { + vm::CellBuilder cbi; + GenericAccount::store_int_message(cbi, gift.destination, gift.gramms < 0 ? 0 : gift.gramms); + if (gift.init_state.not_null()) { + cbi.store_ones(2); + cbi.store_ref(gift.init_state); + } else { + cbi.store_zeroes(1); + } + cbi.store_zeroes(1); + store_gift_message(cbi, gift); + return cbi.finalize(); + } static void store_gift_message(vm::CellBuilder &cb, const Gift &gift) { if (gift.body.not_null()) { auto body = vm::load_cell_slice(gift.body); //TODO: handle error - cb.append_cellslice_bool(body); + CHECK(cb.append_cellslice_bool(body)); return; } diff --git a/crypto/smc-envelope/WalletV3.cpp b/crypto/smc-envelope/WalletV3.cpp index 39993c66..5bd7cfcd 100644 --- a/crypto/smc-envelope/WalletV3.cpp +++ b/crypto/smc-envelope/WalletV3.cpp @@ -62,16 +62,10 @@ td::Ref WalletV3::make_a_gift_message(const td::Ed25519::PrivateKey& p for (auto& gift : gifts) { td::int32 send_mode = 3; - auto gramms = gift.gramms; - if (gramms == -1) { - gramms = 0; + if (gift.gramms == -1) { send_mode += 128; } - vm::CellBuilder cbi; - GenericAccount::store_int_message(cbi, gift.destination, gramms); - store_gift_message(cbi, gift); - auto message_inner = cbi.finalize(); - cb.store_long(send_mode, 8).store_ref(std::move(message_inner)); + cb.store_long(send_mode, 8).store_ref(create_int_message(gift)); } auto message_outer = cb.finalize(); diff --git a/crypto/smc-envelope/WalletV3.h b/crypto/smc-envelope/WalletV3.h index c5cfead3..b3e332dd 100644 --- a/crypto/smc-envelope/WalletV3.h +++ b/crypto/smc-envelope/WalletV3.h @@ -70,3 +70,209 @@ class WalletV3 : public ton::SmartContract, public WalletInterface { td::Result get_public_key_or_throw() const; }; } // namespace ton + +#include "smc-envelope/SmartContractCode.h" +#include "smc-envelope/GenericAccount.h" +#include "block/block-parse.h" +#include +namespace ton { +template +class WalletBase : public SmartContract, public WalletInterface { + public: + using Traits = TraitsT; + using InitData = typename Traits::InitData; + + explicit WalletBase(State state) : SmartContract(std::move(state)) { + } + static td::Ref create(State state) { + return td::Ref(true, std::move(state)); + } + static td::Ref get_init_code(int revision) { + return SmartContractCode::get_code(get_code_type(), revision); + }; + size_t get_max_gifts_size() const override { + return Traits::max_gifts_size; + } + static SmartContractCode::Type get_code_type() { + return Traits::code_type; + } + static td::optional guess_revision(const vm::Cell::Hash& code_hash) { + for (auto i : ton::SmartContractCode::get_revisions(get_code_type())) { + auto code = SmartContractCode::get_code(get_code_type(), i); + if (code->get_hash() == code_hash) { + return i; + } + } + return {}; + } + + static td::Ref create(const InitData& init_data, int revision) { + return td::Ref(true, State{get_init_code(revision), WalletT::get_init_data(init_data)}); + } + + td::Result get_seqno() const { + return TRY_VM([&]() -> td::Result { + Answer answer = this->run_get_method("seqno"); + if (!answer.success) { + return td::Status::Error("seqno get method failed"); + } + return static_cast(answer.stack.write().pop_long_range(std::numeric_limits::max())); + }()); + } + td::Result get_wallet_id() const { + return TRY_VM([&]() -> td::Result { + Answer answer = this->run_get_method("wallet_id"); + if (!answer.success) { + return td::Status::Error("seqno get method failed"); + } + return static_cast(answer.stack.write().pop_long_range(std::numeric_limits::max())); + }()); + } + + td::Result get_balance(td::uint64 account_balance, td::uint32 now) const { + return TRY_VM([&]() -> td::Result { + Answer answer = this->run_get_method(Args().set_method_id("balance").set_balance(account_balance).set_now(now)); + if (!answer.success) { + return td::Status::Error("balance get method failed"); + } + return static_cast(answer.stack.write().pop_long()); + }()); + } + + td::Result get_public_key() const override { + return TRY_VM([&]() -> td::Result { + Answer answer = this->run_get_method("get_public_key"); + if (!answer.success) { + return td::Status::Error("get_public_key get method failed"); + } + auto key_int = answer.stack.write().pop_int(); + LOG(ERROR) << key_int->bit_size(false); + td::SecureString bytes(32); + if (!key_int->export_bytes(bytes.as_mutable_slice().ubegin(), bytes.size(), false)) { + return td::Status::Error("not a public key"); + } + return td::Ed25519::PublicKey(std::move(bytes)); + }()); + }; +}; + +struct RestrictedWalletTraits { + struct InitData { + td::SecureString init_key; + td::SecureString main_key; + td::uint32 wallet_id{0}; + }; + + static constexpr unsigned max_message_size = vm::CellString::max_bytes; + static constexpr unsigned max_gifts_size = 4; + static constexpr auto code_type = SmartContractCode::RestrictedWallet; +}; + +class RestrictedWallet : public WalletBase { + public: + struct Config { + td::uint32 start_at{0}; + std::vector> limits; + }; + + explicit RestrictedWallet(State state) : WalletBase(std::move(state)) { + } + + td::Result get_config() const { + return TRY_VM([this]() -> td::Result { + auto cs = vm::load_cell_slice(get_state().data); + Config config; + td::Ref dict_root; + auto ok = cs.advance(32 + 32 + 256) && cs.fetch_uint_to(32, config.start_at) && cs.fetch_maybe_ref(dict_root); + vm::Dictionary dict(std::move(dict_root), 32); + dict.check_for_each([&](auto cs, auto ptr, auto ptr_bits) { + auto r_seconds = td::narrow_cast_safe(dict.key_as_integer(ptr, true)->to_long()); + if (r_seconds.is_error()) { + ok = false; + return ok; + } + td::uint64 value; + ok &= smc::unpack_grams(cs, value); + config.limits.emplace_back(r_seconds.ok(), value); + return ok; + }); + if (!ok) { + return td::Status::Error("Can't parse config"); + } + std::sort(config.limits.begin(), config.limits.end()); + return config; + }()); + } + + static td::Ref get_init_data(const InitData& init_data) { + vm::CellBuilder cb; + cb.store_long(0, 32); + cb.store_long(init_data.wallet_id, 32); + CHECK(init_data.init_key.size() == 32); + CHECK(init_data.main_key.size() == 32); + cb.store_bytes(init_data.init_key.as_slice()); + cb.store_bytes(init_data.main_key.as_slice()); + return cb.finalize(); + } + + td::Result> get_init_message(const td::Ed25519::PrivateKey& init_private_key, + td::uint32 valid_until, const Config& config) const { + vm::CellBuilder cb; + TRY_RESULT(seqno, get_seqno()); + TRY_RESULT(wallet_id, get_wallet_id()); + LOG(ERROR) << "seqno: " << seqno << " wallet_id: " << wallet_id; + if (seqno != 0) { + return td::Status::Error("Wallet is already inited"); + } + + cb.store_long(wallet_id, 32); + cb.store_long(valid_until, 32); + cb.store_long(seqno, 32); + + cb.store_long(config.start_at, 32); + vm::Dictionary dict(32); + + auto add = [&](td::int32 till, td::uint64 value) { + auto key = dict.integer_key(td::make_refint(till), 32, true); + vm::CellBuilder gcb; + block::tlb::t_Grams.store_integer_value(gcb, td::BigInt256(value)); + dict.set_builder(key.bits(), 32, gcb); + }; + for (auto limit : config.limits) { + add(limit.first, limit.second); + } + cb.store_maybe_ref(dict.get_root_cell()); + + auto message_outer = cb.finalize(); + auto signature = init_private_key.sign(message_outer->get_hash().as_slice()).move_as_ok(); + return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); + } + + td::Result> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until, + td::Span gifts) const override { + CHECK(gifts.size() <= Traits::max_gifts_size); + + vm::CellBuilder cb; + TRY_RESULT(seqno, get_seqno()); + TRY_RESULT(wallet_id, get_wallet_id()); + if (seqno == 0) { + return td::Status::Error("Wallet is not inited yet"); + } + cb.store_long(wallet_id, 32); + cb.store_long(valid_until, 32); + cb.store_long(seqno, 32); + + for (auto& gift : gifts) { + td::int32 send_mode = 3; + if (gift.gramms == -1) { + send_mode += 128; + } + cb.store_long(send_mode, 8).store_ref(create_int_message(gift)); + } + + auto message_outer = cb.finalize(); + auto signature = private_key.sign(message_outer->get_hash().as_slice()).move_as_ok(); + return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); + } +}; +} // namespace ton diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index f143ca0a..361c9947 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -1542,6 +1542,7 @@ template class BenchBocDeserializer : public td::Benchmark { public: BenchBocDeserializer(std::string name, BenchBocDeserializerConfig config) : name_(std::move(name)), config_(config) { + td::PerfWarningTimer perf("A", 1); fast_array_ = vm::FastCompactArray(array_size); td::Random::Xorshift128plus rnd{123}; for (td::uint32 i = 0; i < array_size; i++) { diff --git a/crypto/test/test-smartcont.cpp b/crypto/test/test-smartcont.cpp index 5048bb28..fdce6a82 100644 --- a/crypto/test/test-smartcont.cpp +++ b/crypto/test/test-smartcont.cpp @@ -21,7 +21,9 @@ #include "Ed25519.h" +#include "block/block-auto.h" #include "block/block.h" +#include "block/block-parse.h" #include "fift/Fift.h" #include "fift/words.h" @@ -38,6 +40,7 @@ #include "smc-envelope/WalletV3.h" #include "smc-envelope/HighloadWallet.h" #include "smc-envelope/HighloadWalletV2.h" +#include "smc-envelope/PaymentChannel.h" #include "td/utils/base64.h" #include "td/utils/crypto.h" @@ -489,6 +492,99 @@ TEST(Tonlib, TestGiver) { CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == res->get_hash()); } +TEST(Tonlib, RestrictedWallet) { + //auto source_lookup = fift::create_mem_source_lookup(load_source("smartcont/new-restricted-wallet2.fif")).move_as_ok(); + //source_lookup + //.write_file("/auto/restricted-wallet2-code.fif", load_source("smartcont/auto/restricted-wallet2-code.fif")) + //.ensure(); + //class ZeroOsTime : public fift::OsTime { + //public: + //td::uint32 now() override { + //return 0; + //} + //}; + //source_lookup.set_os_time(std::make_unique()); + //auto priv_key = td::Ed25519::generate_private_key().move_as_ok(); + //auto pub_key = priv_key.get_public_key().move_as_ok(); + //auto pub_key_serialized = block::PublicKey::from_bytes(pub_key.as_octet_string()).move_as_ok().serialize(true); + + //std::vector args = {"path", pub_key_serialized, std::string("100")}; + //auto fift_output = fift::mem_run_fift(std::move(source_lookup), args).move_as_ok(); + + //ton::RestrictedWallet::InitData init_data; + //td::uint64 x = 100 * 1000000000ull; + //init_data.key = &pub_key; + //init_data.start_at = 0; + //init_data.limits = {{-32768, x}, {92, x * 3 / 4}, {183, x * 1 / 2}, {366, x * 1 / 4}, {548, 0}}; + //auto wallet = ton::RestrictedWallet::create(init_data, -1); + + //ASSERT_EQ(0u, wallet->get_seqno().move_as_ok()); + //CHECK(pub_key.as_octet_string() == wallet->get_public_key().move_as_ok().as_octet_string()); + ////LOG(ERROR) << wallet->get_balance(x, 60 * 60 * 24 * 400).move_as_ok(); + + //auto new_wallet_query = fift_output.source_lookup.read_file("rwallet-query.boc").move_as_ok().data; + //auto new_wallet_addr = fift_output.source_lookup.read_file("rwallet.addr").move_as_ok().data; + + //auto address = wallet->get_address(-1); + ////CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32)); + //address.bounceable = false; + //auto res = ton::GenericAccount::create_ext_message(address, wallet->get_init_state(), + //wallet->get_init_message(priv_key).move_as_ok()); + //LOG(ERROR) << "-------"; + //vm::load_cell_slice(res).print_rec(std::cerr); + //LOG(ERROR) << "-------"; + //vm::load_cell_slice(vm::std_boc_deserialize(new_wallet_query).move_as_ok()).print_rec(std::cerr); + //CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == res->get_hash()); + + //auto dest = block::StdAddress::parse("Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX").move_as_ok(); + //fift_output.source_lookup.write_file("/main.fif", load_source("smartcont/wallet-v2.fif")).ensure(); + //fift_output.source_lookup.write_file("rwallet.pk", priv_key.as_octet_string().as_slice()).ensure(); + //fift_output = fift::mem_run_fift( + //std::move(fift_output.source_lookup), + //{"aba", "rwallet", "-C", "TESTv2", "Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX", "0", "321"}) + //.move_as_ok(); + //auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data; + //ton::TestWallet::Gift gift; + //gift.destination = dest; + //gift.message = "TESTv2"; + //gift.gramms = 321000000000ll; + ////CHECK(priv_key.get_public_key().ok().as_octet_string() == wallet->get_public_key().ok().as_octet_string()); + //auto gift_message = ton::GenericAccount::create_ext_message( + //address, {}, wallet->make_a_gift_message(priv_key, 60, {gift}).move_as_ok()); + //LOG(ERROR) << "-------"; + //vm::load_cell_slice(gift_message).print_rec(std::cerr); + //LOG(ERROR) << "-------"; + //vm::load_cell_slice(vm::std_boc_deserialize(wallet_query).move_as_ok()).print_rec(std::cerr); + //CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == gift_message->get_hash()); +} +TEST(Tonlib, RestrictedWallet3) { + auto init_priv_key = td::Ed25519::generate_private_key().move_as_ok(); + auto init_pub_key = init_priv_key.get_public_key().move_as_ok(); + auto priv_key = td::Ed25519::generate_private_key().move_as_ok(); + auto pub_key = priv_key.get_public_key().move_as_ok(); + + ton::RestrictedWallet::InitData init_data; + init_data.init_key = init_pub_key.as_octet_string(); + init_data.main_key = pub_key.as_octet_string(); + init_data.wallet_id = 123; + auto wallet = ton::RestrictedWallet::create(init_data, 1); + + auto address = wallet->get_address(); + + td::uint64 x = 100 * 1000000000ull; + ton::RestrictedWallet::Config config; + config.start_at = 1; + config.limits = {{-32768, x}, {92, x * 3 / 4}, {183, x * 1 / 2}, {366, x * 1 / 4}, {548, 0}}; + CHECK(wallet.write().send_external_message(wallet->get_init_message(init_priv_key, 10, config).move_as_ok()).success); + CHECK(wallet->get_seqno().move_as_ok() == 1); + + ton::WalletInterface::Gift gift; + gift.destination = address; + gift.message = "hello"; + CHECK(wallet.write().send_external_message(wallet->make_a_gift_message(priv_key, 10, {gift}).move_as_ok()).success); + CHECK(wallet->get_seqno().move_as_ok() == 2); +} + class SimpleWallet : public ton::SmartContract { public: SimpleWallet(State state) : SmartContract(std::move(state)) { @@ -1286,3 +1382,397 @@ TEST(Smartcont, DnsManual) { // TODO: rethink semantic of creating an empty dictionary do_dns_test(CheckedDns(true, true)); } + +using namespace ton::pchan; + +template +struct ValidateState { + T& self() { + return static_cast(*this); + } + + void init(td::Ref state) { + state_ = state; + block::gen::ChanData::Record data_rec; + if (!tlb::unpack_cell(state, data_rec)) { + on_fatal_error(td::Status::Error("Expected Data")); + return; + } + if (!tlb::unpack_cell(data_rec.state, self().rec)) { + on_fatal_error(td::Status::Error("Expected StatePayout")); + return; + } + CHECK(self().rec.A.not_null()); + } + + T& expect_grams(td::Ref cs, td::uint64 expected, td::Slice name) { + if (has_fatal_error_) { + return self(); + } + td::RefInt256 got; + CHECK(cs.not_null()); + CHECK(block::tlb::t_Grams.as_integer_to(cs, got)); + if (got->cmp(expected) != 0) { + on_error(td::Status::Error(PSLICE() << name << ": expected " << expected << ", got " << got->to_dec_string())); + } + return self(); + } + template + T& expect_eq(S a, S expected, td::Slice name) { + if (has_fatal_error_) { + return self(); + } + if (!(a == expected)) { + on_error(td::Status::Error(PSLICE() << name << ": expected " << expected << ", got " << a)); + } + return self(); + } + + td::Status finish() { + if (errors_.empty()) { + return td::Status::OK(); + } + std::stringstream ss; + block::gen::t_ChanData.print_ref(ss, state_); + td::StringBuilder sb; + for (auto& error : errors_) { + sb << error << "\n"; + } + sb << ss.str(); + return td::Status::Error(sb.as_cslice()); + } + + void on_fatal_error(td::Status error) { + CHECK(!has_fatal_error_); + has_fatal_error_ = true; + on_error(std::move(error)); + } + void on_error(td::Status error) { + CHECK(error.is_error()); + errors_.push_back(std::move(error)); + } + + public: + td::Ref state_; + bool has_fatal_error_{false}; + std::vector errors_; +}; + +struct ValidateStatePayout : public ValidateState { + ValidateStatePayout& expect_A(td::uint64 a) { + expect_grams(rec.A, a, "A"); + return *this; + } + ValidateStatePayout& expect_B(td::uint64 b) { + expect_grams(rec.B, b, "B"); + return *this; + } + + ValidateStatePayout(td::Ref state) { + init(std::move(state)); + } + + block::gen::ChanState::Record_chan_state_payout rec; +}; + +struct ValidateStateInit : public ValidateState { + ValidateStateInit& expect_A(td::uint64 a) { + expect_grams(rec.A, a, "A"); + return *this; + } + ValidateStateInit& expect_B(td::uint64 b) { + expect_grams(rec.B, b, "B"); + return *this; + } + ValidateStateInit& expect_min_A(td::uint64 a) { + expect_grams(rec.min_A, a, "min_A"); + return *this; + } + ValidateStateInit& expect_min_B(td::uint64 b) { + expect_grams(rec.min_B, b, "min_B"); + return *this; + } + ValidateStateInit& expect_expire_at(td::uint32 b) { + expect_eq(rec.expire_at, b, "expire_at"); + return *this; + } + ValidateStateInit& expect_signed_A(bool x) { + expect_eq(rec.signed_A, x, "signed_A"); + return *this; + } + ValidateStateInit& expect_signed_B(bool x) { + expect_eq(rec.signed_B, x, "signed_B"); + return *this; + } + + ValidateStateInit(td::Ref state) { + init(std::move(state)); + } + + block::gen::ChanState::Record_chan_state_init rec; +}; + +struct ValidateStateClose : public ValidateState { + ValidateStateClose& expect_A(td::uint64 a) { + expect_grams(rec.A, a, "A"); + return *this; + } + ValidateStateClose& expect_B(td::uint64 b) { + expect_grams(rec.B, b, "B"); + return *this; + } + ValidateStateClose& expect_promise_A(td::uint64 a) { + expect_grams(rec.promise_A, a, "promise_A"); + return *this; + } + ValidateStateClose& expect_promise_B(td::uint64 b) { + expect_grams(rec.promise_B, b, "promise_B"); + return *this; + } + ValidateStateClose& expect_expire_at(td::uint32 b) { + expect_eq(rec.expire_at, b, "expire_at"); + return *this; + } + ValidateStateClose& expect_signed_A(bool x) { + expect_eq(rec.signed_A, x, "signed_A"); + return *this; + } + ValidateStateClose& expect_signed_B(bool x) { + expect_eq(rec.signed_B, x, "signed_B"); + return *this; + } + + ValidateStateClose(td::Ref state) { + init(std::move(state)); + } + + block::gen::ChanState::Record_chan_state_close rec; +}; + +// config$_ initTimeout:int exitTimeout:int a_key:int256 b_key:int256 a_addr b_addr channel_id:int256 = Config; +TEST(Smarcont, Channel) { + auto code = ton::SmartContractCode::get_code(ton::SmartContractCode::PaymentChannel); + Config config; + auto a_pkey = td::Ed25519::generate_private_key().move_as_ok(); + auto b_pkey = td::Ed25519::generate_private_key().move_as_ok(); + config.init_timeout = 20; + config.close_timeout = 40; + auto dest = block::StdAddress::parse("Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX").move_as_ok(); + config.a_addr = dest; + config.b_addr = dest; + config.a_key = a_pkey.get_public_key().ok().as_octet_string(); + config.b_key = b_pkey.get_public_key().ok().as_octet_string(); + config.channel_id = 123; + + Data data; + data.config = config.serialize(); + data.state = data.init_state(); + auto data_cell = data.serialize(); + + auto channel = ton::SmartContract::create(ton::SmartContract::State{code, data_cell}); + ValidateStateInit(channel->get_state().data) + .expect_A(0) + .expect_B(0) + .expect_min_A(0) + .expect_min_B(0) + .expect_signed_A(false) + .expect_signed_B(false) + .expect_expire_at(0) + .finish() + .ensure(); + + enum err { + ok = 0, + wrong_a_signature = 31, + wrong_b_signature, + msg_value_too_small, + replay_protection, + no_timeout, + expected_init, + expected_close, + no_promise_signature, + wrong_channel_id + }; + +#define expect_code(description, expected_code, e) \ + { \ + auto res = e; \ + LOG_IF(FATAL, expected_code != res.code) << " res.code=" << res.code << " " << description << "\n" << #e; \ + } +#define expect_ok(description, e) expect_code(description, 0, e) + + expect_code("Trying to invoke a timeout while channel is empty", no_timeout, + channel.write().send_external_message(MsgTimeoutBuilder().finalize(), + ton::SmartContract::Args().set_now(1000000))); + + expect_code("External init message with no signatures", replay_protection, + channel.write().send_external_message(MsgInitBuilder().channel_id(config.channel_id).finalize())); + expect_code("Internal init message with not enough value", msg_value_too_small, + channel.write().send_internal_message( + MsgInitBuilder().channel_id(config.channel_id).inc_A(1000).min_B(2000).with_a_key(&a_pkey).finalize(), + ton::SmartContract::Args().set_amount(100))); + expect_code( + "Internal init message with wrong channel_id", wrong_channel_id, + channel.write().send_internal_message(MsgInitBuilder().inc_A(1000).min_B(2000).with_a_key(&a_pkey).finalize(), + ton::SmartContract::Args().set_amount(1000))); + expect_ok("A init with (inc_A = 1000, min_A = 1, min_B = 2000)", + channel.write().send_internal_message(MsgInitBuilder() + .channel_id(config.channel_id) + .inc_A(1000) + .min_A(1) + .min_B(2000) + .with_a_key(&a_pkey) + .finalize(), + ton::SmartContract::Args().set_amount(1000))); + ValidateStateInit(channel->get_state().data) + .expect_A(1000) + .expect_B(0) + .expect_min_A(1) + .expect_min_B(2000) + .expect_signed_A(true) + .expect_signed_B(false) + .expect_expire_at(config.init_timeout) + .finish() + .ensure(); + + expect_code("Repeated init of A init with (inc_A = 100, min_B = 5000). Must be ignored", replay_protection, + channel.write().send_internal_message( + MsgInitBuilder().channel_id(config.channel_id).inc_A(100).min_B(5000).with_a_key(&a_pkey).finalize(), + ton::SmartContract::Args().set_amount(1000))); + expect_code( + "Trying to invoke a timeout too early", no_timeout, + channel.write().send_external_message(MsgTimeoutBuilder().finalize(), ton::SmartContract::Args().set_now(0))); + + { + auto channel_copy = channel; + expect_ok("Invoke a timeout", channel_copy.write().send_external_message(MsgTimeoutBuilder().finalize(), + ton::SmartContract::Args().set_now(21))); + ValidateStatePayout(channel_copy->get_state().data).expect_A(1000).expect_B(0).finish().ensure(); + } + { + auto channel_copy = channel; + expect_ok("B init with inc_B < min_B. Leads to immediate payout", + channel_copy.write().send_internal_message( + MsgInitBuilder().channel_id(config.channel_id).inc_B(1500).with_b_key(&b_pkey).finalize(), + ton::SmartContract::Args().set_amount(1500))); + ValidateStatePayout(channel_copy->get_state().data).expect_A(1000).expect_B(1500).finish().ensure(); + } + + expect_ok("B init with (inc_B = 2000, min_A = 1, min_A = 1000)", + channel.write().send_internal_message( + MsgInitBuilder().channel_id(config.channel_id).inc_B(2000).min_A(1000).with_b_key(&b_pkey).finalize(), + ton::SmartContract::Args().set_amount(2000))); + ValidateStateClose(channel->get_state().data) + .expect_A(1000) + .expect_B(2000) + .expect_promise_A(0) + .expect_promise_B(0) + .expect_signed_A(false) + .expect_signed_B(false) + .expect_expire_at(0) + .finish() + .ensure(); + + { + auto channel_copy = channel; + expect_ok("A&B send Promise(1000000, 1000000 + 10) signed by nobody", + channel_copy.write().send_external_message(MsgCloseBuilder() + .signed_promise(SignedPromiseBuilder() + .promise_A(1000000) + .promise_B(1000000 + 10) + .channel_id(config.channel_id) + .finalize()) + .with_a_key(&a_pkey) + .with_b_key(&b_pkey) + .finalize(), + ton::SmartContract::Args().set_now(21))); + ValidateStatePayout(channel_copy->get_state().data).expect_A(1000 + 10).expect_B(2000 - 10).finish().ensure(); + } + { + auto channel_copy = channel; + expect_ok("A&B send Promise(1000000, 1000000 + 10) signed by A", + channel_copy.write().send_external_message(MsgCloseBuilder() + .signed_promise(SignedPromiseBuilder() + .promise_A(1000000) + .promise_B(1000000 + 10) + .with_key(&a_pkey) + .channel_id(config.channel_id) + .finalize()) + .with_a_key(&a_pkey) + .with_b_key(&b_pkey) + .finalize(), + ton::SmartContract::Args().set_now(21))); + ValidateStatePayout(channel_copy->get_state().data).expect_A(1000 + 10).expect_B(2000 - 10).finish().ensure(); + } + + expect_code( + "A sends Promise(1000000, 0) signed by A", wrong_b_signature, + channel.write().send_external_message( + MsgCloseBuilder() + .signed_promise( + SignedPromiseBuilder().promise_A(1000000).with_key(&a_pkey).channel_id(config.channel_id).finalize()) + .with_a_key(&a_pkey) + .finalize(), + ton::SmartContract::Args().set_now(21))); + expect_code( + "B sends Promise(1000000, 0) signed by B", wrong_a_signature, + channel.write().send_external_message( + MsgCloseBuilder() + .signed_promise( + SignedPromiseBuilder().promise_A(1000000).with_key(&b_pkey).channel_id(config.channel_id).finalize()) + .with_b_key(&b_pkey) + .finalize(), + ton::SmartContract::Args().set_now(21))); + expect_code("B sends Promise(1000000, 0) signed by A with wrong channel_id", wrong_channel_id, + channel.write().send_external_message(MsgCloseBuilder() + .signed_promise(SignedPromiseBuilder() + .promise_A(1000000) + .with_key(&a_pkey) + .channel_id(config.channel_id + 1) + .finalize()) + .with_b_key(&b_pkey) + .finalize(), + ton::SmartContract::Args().set_now(21))); + expect_code( + "B sends unsigned Promise(1000000, 0)", no_promise_signature, + channel.write().send_external_message( + MsgCloseBuilder() + .signed_promise(SignedPromiseBuilder().promise_A(1000000).channel_id(config.channel_id).finalize()) + .with_b_key(&b_pkey) + .finalize(), + ton::SmartContract::Args().set_now(21))); + + expect_ok( + "B sends Promise(1000000, 0) signed by A", + channel.write().send_external_message( + MsgCloseBuilder() + .signed_promise( + SignedPromiseBuilder().promise_A(1000000).with_key(&a_pkey).channel_id(config.channel_id).finalize()) + .with_b_key(&b_pkey) + .finalize(), + ton::SmartContract::Args().set_now(21))); + ValidateStateClose(channel->get_state().data) + .expect_A(1000) + .expect_B(2000) + .expect_promise_A(1000000) + .expect_promise_B(0) + .expect_signed_A(false) + .expect_signed_B(true) + .expect_expire_at(21 + config.close_timeout) + .finish() + .ensure(); + + expect_ok("B sends Promise(0, 1000000 + 10) signed by A", + channel.write().send_external_message(MsgCloseBuilder() + .signed_promise(SignedPromiseBuilder() + .promise_B(1000000 + 10) + .with_key(&b_pkey) + .channel_id(config.channel_id) + .finalize()) + .with_a_key(&a_pkey) + .finalize(), + ton::SmartContract::Args().set_now(21))); + ValidateStatePayout(channel->get_state().data).expect_A(1000 + 10).expect_B(2000 - 10).finish().ensure(); +#undef expect_ok +#undef expect_code +} diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 0d539c74..916fc023 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -19,6 +19,7 @@ #include "vm/cells/CellSlice.h" #include "vm/excno.hpp" #include "td/utils/bits.h" +#include "td/utils/misc.h" namespace vm { @@ -719,6 +720,10 @@ bool CellSlice::prefetch_bytes(unsigned char* buffer, unsigned bytes) const { } } +bool CellSlice::fetch_bytes(td::MutableSlice slice) { + return fetch_bytes(slice.ubegin(), td::narrow_cast(slice.size())); +} + bool CellSlice::fetch_bytes(unsigned char* buffer, unsigned bytes) { if (prefetch_bytes(buffer, bytes)) { advance(bytes * 8); @@ -728,6 +733,10 @@ bool CellSlice::fetch_bytes(unsigned char* buffer, unsigned bytes) { } } +bool CellSlice::prefetch_bytes(td::MutableSlice slice) const { + return prefetch_bytes(slice.ubegin(), td::narrow_cast(slice.size())); +} + Ref CellSlice::prefetch_ref(unsigned offset) const { if (offset < size_refs()) { auto ref_id = refs_st + offset; diff --git a/crypto/vm/cells/CellSlice.h b/crypto/vm/cells/CellSlice.h index a2e528d6..33fad741 100644 --- a/crypto/vm/cells/CellSlice.h +++ b/crypto/vm/cells/CellSlice.h @@ -218,7 +218,9 @@ class CellSlice : public td::CntObject { return prefetch_bits_to(buffer.bits(), n); } bool fetch_bytes(unsigned char* buffer, unsigned bytes); + bool fetch_bytes(td::MutableSlice slice); bool prefetch_bytes(unsigned char* buffer, unsigned bytes) const; + bool prefetch_bytes(td::MutableSlice slice) const; td::BitSlice as_bitslice() const { return prefetch_bits(size()); } diff --git a/crypto/vm/cells/MerkleProof.cpp b/crypto/vm/cells/MerkleProof.cpp index a1c4131f..aee34367 100644 --- a/crypto/vm/cells/MerkleProof.cpp +++ b/crypto/vm/cells/MerkleProof.cpp @@ -148,6 +148,11 @@ class MerkleProofCombineFast { MerkleProofCombineFast(Ref a, Ref b) : a_(std::move(a)), b_(std::move(b)) { } td::Result> run() { + if (a_.is_null()) { + return b_; + } else if (b_.is_null()) { + return a_; + } TRY_RESULT_ASSIGN(a_, unpack_proof(a_)); TRY_RESULT_ASSIGN(b_, unpack_proof(b_)); TRY_RESULT(res, run_raw()); @@ -204,6 +209,11 @@ class MerkleProofCombine { MerkleProofCombine(Ref a, Ref b) : a_(std::move(a)), b_(std::move(b)) { } td::Result> run() { + if (a_.is_null()) { + return b_; + } else if (b_.is_null()) { + return a_; + } TRY_RESULT_ASSIGN(a_, unpack_proof(a_)); TRY_RESULT_ASSIGN(b_, unpack_proof(b_)); TRY_RESULT(res, run_raw()); @@ -323,6 +333,10 @@ Ref MerkleProof::combine(Ref a, Ref b) { return res.move_as_ok(); } +td::Result> MerkleProof::combine_status(Ref a, Ref b) { + return MerkleProofCombine(std::move(a), std::move(b)).run(); +} + Ref MerkleProof::combine_fast(Ref a, Ref b) { auto res = MerkleProofCombineFast(std::move(a), std::move(b)).run(); if (res.is_error()) { @@ -331,6 +345,10 @@ Ref MerkleProof::combine_fast(Ref a, Ref b) { return res.move_as_ok(); } +td::Result> MerkleProof::combine_fast_status(Ref a, Ref b) { + return MerkleProofCombineFast(std::move(a), std::move(b)).run(); +} + Ref MerkleProof::combine_raw(Ref a, Ref b) { auto res = MerkleProofCombine(std::move(a), std::move(b)).run_raw(); if (res.is_error()) { diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index cb50f1d0..4f2add6c 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -38,7 +38,9 @@ class MerkleProof { static Ref virtualize(Ref cell, int virtualization); static Ref combine(Ref a, Ref b); + static td::Result> combine_status(Ref a, Ref b); static Ref combine_fast(Ref a, Ref b); + static td::Result> combine_fast_status(Ref a, Ref b); // works with upwrapped proofs // works fine with cell of non-zero level, but this is not supported (yet?) in MerkeProof special cell diff --git a/crypto/vm/db/TonDb.cpp b/crypto/vm/db/TonDb.cpp index 68f677a1..ac69973f 100644 --- a/crypto/vm/db/TonDb.cpp +++ b/crypto/vm/db/TonDb.cpp @@ -255,15 +255,15 @@ TonDbTransactionImpl::TonDbTransactionImpl(std::shared_ptr kv) : kv_(s } void TonDbTransactionImpl::begin() { - kv_->begin_transaction(); + kv_->begin_write_batch(); generation_++; } void TonDbTransactionImpl::commit() { - kv_->commit_transaction(); + kv_->commit_write_batch(); reader_.reset(kv_->snapshot().release()); } void TonDbTransactionImpl::abort() { - kv_->abort_transaction(); + kv_->abort_write_batch(); } void TonDbTransactionImpl::clear_cache() { contracts_ = {}; diff --git a/crypto/vm/excno.hpp b/crypto/vm/excno.hpp index 31742e08..3be48168 100644 --- a/crypto/vm/excno.hpp +++ b/crypto/vm/excno.hpp @@ -77,6 +77,13 @@ class VmError { long long get_arg() const { return arg; } + td::Status as_status() const { + return td::Status::Error(td::Slice{get_msg()}); + } + template + td::Status as_status(T pfx) const { + return td::Status::Error(PSLICE() << pfx << get_msg()); + } }; struct VmNoGas { @@ -90,6 +97,13 @@ struct VmNoGas { operator VmError() const { return VmError{Excno::out_of_gas, "out of gas"}; } + td::Status as_status() const { + return td::Status::Error(td::Slice{get_msg()}); + } + template + td::Status as_status(T pfx) const { + return td::Status::Error(PSLICE() << pfx << get_msg()); + } }; struct VmVirtError { @@ -106,6 +120,13 @@ struct VmVirtError { operator VmError() const { return VmError{Excno::virt_err, "prunned branch", virtualization}; } + td::Status as_status() const { + return td::Status::Error(td::Slice{get_msg()}); + } + template + td::Status as_status(T pfx) const { + return td::Status::Error(PSLICE() << pfx << get_msg()); + } }; struct VmFatal {}; @@ -114,12 +135,12 @@ template auto try_f(F&& f) noexcept -> decltype(f()) { try { return f(); - } catch (vm::VmError error) { - return td::Status::Error(PSLICE() << "Got a vm exception: " << error.get_msg()); - } catch (vm::VmVirtError error) { - return td::Status::Error(PSLICE() << "Got a vm virtualization exception: " << error.get_msg()); - } catch (vm::VmNoGas error) { - return td::Status::Error(PSLICE() << "Got a vm no gas exception: " << error.get_msg()); + } catch (vm::VmError& error) { + return error.as_status("Got a vm exception: "); + } catch (vm::VmVirtError& error) { + return error.as_status("Got a vm virtualization exception: "); + } catch (vm::VmNoGas& error) { + return error.as_status("Got a vm no gas exception: "); } } diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index f1f90f42..98176ba6 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -135,8 +135,36 @@ void TestNode::run() { ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_, make_callback()); } -void TestNode::got_result() { +void TestNode::got_result(td::Result R, td::Promise promise) { + if (R.is_error()) { + auto err = R.move_as_error(); + LOG(ERROR) << "failed query: " << err; + promise.set_error(std::move(err)); + td::actor::send_closure_later(actor_id(this), &TestNode::after_got_result, false); + return; + } + auto data = R.move_as_ok(); + auto F = ton::fetch_tl_object(data.clone(), true); + if (F.is_ok()) { + auto f = F.move_as_ok(); + auto err = td::Status::Error(f->code_, f->message_); + LOG(ERROR) << "liteserver error: " << err; + promise.set_error(std::move(err)); + td::actor::send_closure_later(actor_id(this), &TestNode::after_got_result, false); + return; + } + promise.set_result(std::move(data)); + td::actor::send_closure_later(actor_id(this), &TestNode::after_got_result, true); +} + +void TestNode::after_got_result(bool ok) { running_queries_--; + if (ex_mode_ && !ok) { + LOG(ERROR) << "fatal error executing command-line queries, skipping the rest"; + std::cout.flush(); + std::cerr.flush(); + std::_Exit(1); + } if (!running_queries_ && ex_queries_.size() > 0) { auto data = std::move(ex_queries_[0]); ex_queries_.erase(ex_queries_.begin()); @@ -152,28 +180,12 @@ void TestNode::got_result() { bool TestNode::envelope_send_query(td::BufferSlice query, td::Promise promise) { running_queries_++; if (!ready_ || client_.empty()) { - LOG(ERROR) << "failed to send query to server: not ready"; + got_result(td::Status::Error("failed to send query to server: not ready"), std::move(promise)); return false; } auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - auto err = R.move_as_error(); - LOG(ERROR) << "failed query: " << err; - promise.set_error(std::move(err)); - return; - } - auto data = R.move_as_ok(); - auto F = ton::fetch_tl_object(data.clone(), true); - if (F.is_ok()) { - auto f = F.move_as_ok(); - auto err = td::Status::Error(f->code_, f->message_); - LOG(ERROR) << "received error: " << err; - promise.set_error(std::move(err)); - return; - } - promise.set_result(std::move(data)); - td::actor::send_closure(SelfId, &TestNode::got_result); + td::actor::send_closure(SelfId, &TestNode::got_result, std::move(R), std::move(promise)); }); td::BufferSlice b = ton::serialize_tl_object(ton::create_tl_object(std::move(query)), true); @@ -306,7 +318,7 @@ bool TestNode::get_server_time() { LOG(ERROR) << "cannot parse answer to liteServer.getTime"; } else { server_time_ = F.move_as_ok()->now_; - server_time_got_at_ = static_cast(td::Clocks::system()); + server_time_got_at_ = now(); LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; } } @@ -355,7 +367,7 @@ void TestNode::set_server_version(td::int32 version, td::int64 capabilities) { void TestNode::set_server_time(int server_utime) { server_time_ = server_utime; - server_time_got_at_ = static_cast(td::Clocks::system()); + server_time_got_at_ = now(); LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; } @@ -425,8 +437,7 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt } td::TerminalIO::out() << "latest masterchain block known to server is " << blkid.to_str(); if (created > 0) { - td::TerminalIO::out() << " created at " << created << " (" << static_cast(td::Clocks::system()) - created - << " seconds ago)\n"; + td::TerminalIO::out() << " created at " << created << " (" << now() - created << " seconds ago)\n"; } else { td::TerminalIO::out() << "\n"; } @@ -898,6 +909,8 @@ bool TestNode::show_help(std::string command) { "getconfig [...]\tShows specified or all configuration parameters from the latest masterchain state\n" "getconfigfrom [...]\tShows specified or all configuration parameters from the " "masterchain state of \n" + "getkeyconfig [...]\tShows specified or all configuration parameters from the " + "previous key block with respect to \n" "saveconfig []\tSaves all configuration parameters into specified file\n" "gethead \tShows block header for \n" "getblock \tDownloads block\n" @@ -924,6 +937,14 @@ bool TestNode::show_help(std::string command) { "recentcreatorstats [ []]\tLists block creator statistics " "updated after by validator public " "key\n" + "checkload[all|severe] []\tChecks whether all validators worked " + "properly during specified time " + "interval, and optionally saves proofs into -.boc\n" + "loadproofcheck \tChecks a validator misbehavior proof previously created by checkload\n" + "pastvalsets\tLists known past validator set ids and their hashes\n" + "savecomplaints \tSaves all complaints registered for specified validator set id " + "into files .boc\n" + "complaintprice \tComputes the price (in nanograms) for creating a complaint\n" "known\tShows the list of all known block ids\n" "knowncells\tShows the list of hashes of all known (cached) cells\n" "dumpcell \nDumps a cached cell by a prefix of its hash\n" @@ -989,10 +1010,12 @@ bool TestNode::do_parse_line() { blkid = mc_last_id_; std::string filename; return get_word_to(filename) && (seekeoln() || parse_block_id_ext(blkid)) && seekeoln() && - get_config_params(blkid, trivial_promise(), -1, filename); + parse_get_config_params(blkid, -1, filename); } else if (word == "getconfig" || word == "getconfigfrom") { blkid = mc_last_id_; - return (word == "getconfig" || parse_block_id_ext(blkid)) && get_config_params(blkid, trivial_promise(), 0); + return (word == "getconfig" || parse_block_id_ext(blkid)) && parse_get_config_params(blkid, 0); + } else if (word == "getkeyconfig") { + return parse_block_id_ext(blkid) && parse_get_config_params(blkid, 0x8000); } else if (word == "getblock") { return parse_block_id_ext(blkid) && seekeoln() && get_block(blkid, false); } else if (word == "dumpblock") { @@ -1002,7 +1025,7 @@ bool TestNode::do_parse_line() { } else if (word == "dumpstate") { return parse_block_id_ext(blkid) && seekeoln() && get_state(blkid, true); } else if (word == "gethead") { - return parse_block_id_ext(blkid) && seekeoln() && get_block_header(blkid, 0xffff); + return parse_block_id_ext(blkid) && seekeoln() && get_show_block_header(blkid, 0xffff); } else if (word == "dumptrans") { return parse_block_id_ext(blkid) && parse_account_addr(workchain, addr) && parse_lt(lt) && seekeoln() && get_one_transaction(blkid, workchain, addr, lt, true); @@ -1022,17 +1045,36 @@ bool TestNode::do_parse_line() { return parse_block_id_ext(blkid) && (seekeoln() || parse_block_id_ext(blkid2)) && seekeoln() && get_block_proof(blkid, blkid2, blkid2.is_valid() + (word == "blkproofchain") * 0x1000); } else if (word == "byseqno") { - return parse_shard_id(shard) && parse_uint32(seqno) && seekeoln() && lookup_block(shard, 1, seqno); + return parse_shard_id(shard) && parse_uint32(seqno) && seekeoln() && lookup_show_block(shard, 1, seqno); } else if (word == "byutime") { - return parse_shard_id(shard) && parse_uint32(utime) && seekeoln() && lookup_block(shard, 4, utime); + return parse_shard_id(shard) && parse_uint32(utime) && seekeoln() && lookup_show_block(shard, 4, utime); } else if (word == "bylt") { - return parse_shard_id(shard) && parse_lt(lt) && seekeoln() && lookup_block(shard, 2, lt); + return parse_shard_id(shard) && parse_lt(lt) && seekeoln() && lookup_show_block(shard, 2, lt); } else if (word == "creatorstats" || word == "recentcreatorstats") { count = 1000; int mode = (word == "recentcreatorstats" ? 4 : 0); - return parse_block_id_ext(blkid) && (!mode || parse_uint32(utime)) && (seekeoln() || parse_uint32(count)) && - (seekeoln() || (parse_hash(hash) && (mode |= 1))) && seekeoln() && - get_creator_stats(blkid, mode, count, hash, utime); + return parse_block_id_ext(blkid) && (!mode || parse_uint32(utime)) && + (seekeoln() ? (mode |= 0x100) : parse_uint32(count)) && (seekeoln() || (parse_hash(hash) && (mode |= 1))) && + seekeoln() && get_creator_stats(blkid, mode, count, hash, utime); + } else if (word == "checkload" || word == "checkloadall" || word == "checkloadsevere") { + int time1, time2, mode = (word == "checkloadsevere"); + std::string file_pfx; + return parse_int32(time1) && parse_int32(time2) && (seekeoln() || ((mode |= 2) && get_word_to(file_pfx))) && + seekeoln() && check_validator_load(time1, time2, mode, file_pfx); + } else if (word == "loadproofcheck") { + std::string filename; + return get_word_to(filename) && seekeoln() && set_error(check_validator_load_proof(filename)); + } else if (word == "pastvalsets") { + return eoln() && get_past_validator_sets(); + } else if (word == "savecomplaints") { + td::uint32 elect_id; + std::string file_pfx; + return parse_uint32(elect_id) && get_word_to(file_pfx) && seekeoln() && get_complaints(elect_id, file_pfx); + } else if (word == "complaintprice") { + td::uint32 expire_in; + std::string filename; + return parse_uint32(expire_in) && get_word_to(filename) && seekeoln() && + set_error(get_complaint_price(expire_in, filename)); } else if (word == "known") { return eoln() && show_new_blkids(true); } else if (word == "knowncells") { @@ -1251,6 +1293,209 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a } } +bool TestNode::get_elector_addr(td::Promise promise) { + if (elect_addr_queried_) { + promise.set_result(elect_addr_); + return true; + } + auto P = td::PromiseCreator::lambda( + [this, promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, config, std::move(R), "cannot obtain elector address from configuration:"); + if (elect_addr_queried_) { + promise.set_result(elect_addr_); + } else { + promise.set_error(td::Status::Error("cannot obtain elector address from configuration parameter #1")); + } + }); + return get_config_params(mc_last_id_, std::move(P), 0x3000, "", {1}); +} + +bool TestNode::get_past_validator_sets() { + return get_elector_addr([this](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << res.move_as_error(); + } else { + send_past_vset_query(res.move_as_ok()); + } + }); +} + +bool TestNode::send_past_vset_query(ton::StdSmcAddress elector_addr) { + std::vector params; + auto P = td::PromiseCreator::lambda([this](td::Result> R) { + if (R.is_error()) { + LOG(ERROR) << R.move_as_error(); + return; + } + auto S = R.move_as_ok(); + if (S.size() < 1 || !S.back().is_list()) { + LOG(ERROR) << "past_elections_list did not return a value of type tuple"; + return; + } + register_past_vset_info(std::move(S.back())); + }); + return start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "past_elections_list", std::move(params), 0x1f, + std::move(P)); +} + +void TestNode::register_past_vset_info(vm::StackEntry list) { + try { + while (!list.empty()) { + auto tup = std::move(list).as_tuple_range(2, 2); + if (tup.is_null()) { + LOG(ERROR) << "invalid list in the result of past_elections_list"; + return; + } + list = tup->at(1); + auto t2 = tup->at(0).as_tuple_range(255, 3); + if (t2.is_null()) { + LOG(ERROR) << "invalid list entry in the result of past_elections_list"; + return; + } + auto x = t2->at(0).as_int(), y = t2->at(2).as_int(); + if (x.is_null() || y.is_null() || !x->unsigned_fits_bits(32) || !y->unsigned_fits_bits(256)) { + LOG(ERROR) << "invalid components in a list entry in the result of past_elections_list"; + return; + } + td::TerminalIO::out() << "PAST_VSET\t" << td::dec_string(x) << "\t" << td::hex_string(y, true, 64) << std::endl; + } + } catch (vm::VmError& err) { + LOG(ERROR) << "vm error while scanning result: " << err.get_msg(); + } +} + +bool TestNode::get_complaints(unsigned elect_id, std::string file_pfx) { + return get_elector_addr([this, elect_id, file_pfx](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << res.move_as_error(); + } else { + send_get_complaints_query(elect_id, res.move_as_ok(), file_pfx); + } + }); +} + +void TestNode::send_get_complaints_query(unsigned elect_id, ton::StdSmcAddress elector_addr, std::string file_pfx) { + std::vector params; + params.emplace_back(td::make_refint(elect_id)); + auto P = td::PromiseCreator::lambda([this, elect_id, file_pfx](td::Result> R) { + if (R.is_error()) { + LOG(ERROR) << R.move_as_error(); + return; + } + auto S = R.move_as_ok(); + if (S.size() < 1 || !(S.back().empty() || S.back().is_cell())) { + LOG(ERROR) << "get_past_complaints did not return a value of type cell"; + return; + } + try { + save_complaints(elect_id, std::move(S.back()).as_cell(), file_pfx); + } catch (vm::VmError& err) { + LOG(ERROR) << "vm error: " << err.get_msg(); + } catch (vm::VmVirtError& err) { + LOG(ERROR) << "vm virtualization error: " << err.get_msg(); + } + }); + start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "get_past_complaints", std::move(params), 0x1f, + std::move(P)); +} + +void TestNode::save_complaints(unsigned elect_id, Ref complaints, std::string file_pfx) { + vm::Dictionary dict{std::move(complaints), 256}; + for (auto entry : dict) { + block::gen::ValidatorComplaintStatus::Record rec; + block::gen::ValidatorComplaint::Record rec2; + if (!(tlb::csr_unpack(std::move(entry.second), rec) && tlb::unpack_cell(rec.complaint, rec2))) { + LOG(ERROR) << "cannot unpack complaint with key " << entry.first.to_hex(256); + break; + } + if (rec.complaint->get_hash().as_bitslice() != entry.first) { + LOG(ERROR) << "invalid complaint key " << entry.first.to_hex(256) << ": actual complaint hash is " + << rec.complaint->get_hash().to_hex(); + break; + } + std::string filename = file_pfx + entry.first.to_hex(256) + ".boc"; + auto boc = vm::std_boc_serialize(rec.complaint, 2); + if (boc.is_error()) { + LOG(ERROR) << "cannot serialize complaint"; + break; + } + auto len = boc.ok().size(); + auto res1 = td::write_file(filename, boc.move_as_ok()); + if (res1.is_error()) { + LOG(ERROR) << "cannot save serialized complaint to file `" << filename << "` : " << res1.move_as_error(); + return; + } + LOG(DEBUG) << "saved " << len << " bytes into file `" << filename << "`"; + td::TerminalIO::out() << "SAVE_COMPLAINT\t" << elect_id << '\t' << entry.first.to_hex(256) << '\t' + << rec2.validator_pubkey.to_hex() << '\t' << rec2.created_at << '\t' << filename; + } +} + +td::Status TestNode::get_complaint_price(unsigned expires_in, std::string filename) { + LOG(DEBUG) << "reading complaint file " << filename; + TRY_RESULT_PREFIX(data, td::read_file(filename), "cannot read complaint file:"); + TRY_RESULT_PREFIX(complaint, vm::std_boc_deserialize(data), + PSLICE() << "cannot deserialize bag-of-cells read from complaint file `" << filename << "`:"); + if (complaint.is_null()) { + return td::Status::Error("complaint is null"); + } + block::gen::ValidatorComplaint::Record rec; + if (!tlb::unpack_cell(complaint, rec)) { + return td::Status::Error("cannot deserialize complaint"); + } + td::Bits256 chash = complaint->get_hash().bits(); + vm::VmStorageStat stat{1 << 22}; + if (!stat.add_storage(std::move(complaint))) { + return td::Status::Error("cannot compute storage size for this complaint"); + } + return get_complaint_price(expires_in, (unsigned)stat.bits, (unsigned)stat.refs, chash, filename); +} + +td::Status TestNode::get_complaint_price(unsigned expires_in, unsigned bits, unsigned refs, td::Bits256 chash, + std::string filename) { + LOG(INFO) << "complaint `" << filename << "`: " << bits << " bits, " << refs << " references"; + return get_elector_addr([this, filename, expires_in, bits, refs, chash](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << res.move_as_error(); + } else { + send_compute_complaint_price_query(res.move_as_ok(), expires_in, bits, refs, chash, filename); + } + }) + ? td::Status::OK() + : td::Status::Error("cannot obtain elector address"); +} + +void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_addr, unsigned expires_in, unsigned bits, + unsigned refs, td::Bits256 chash, std::string filename) { + std::vector params; + params.emplace_back(td::make_refint(bits)); + params.emplace_back(td::make_refint(refs)); + params.emplace_back(td::make_refint(expires_in)); + auto P = td::PromiseCreator::lambda( + [this, expires_in, bits, refs, chash, filename](td::Result> R) { + if (R.is_error()) { + LOG(ERROR) << R.move_as_error(); + return; + } + auto S = R.move_as_ok(); + if (S.size() < 1 || !S.back().is_int()) { + LOG(ERROR) << "complaint_storage_price did not return a value of type cell"; + return; + } + try { + auto price = std::move(S.back()).as_int(); + td::TerminalIO::out() << "COMPLAINT_PRICE\t" << chash.to_hex() << '\t' << td::dec_string(price) << '\t' + << bits << '\t' << refs << '\t' << expires_in << '\t' << filename << std::endl; + } catch (vm::VmError& err) { + LOG(ERROR) << "vm error: " << err.get_msg(); + } catch (vm::VmVirtError& err) { + LOG(ERROR) << "vm virtualization error: " << err.get_msg(); + } + }); + start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "complaint_storage_price", std::move(params), 0x1f, + std::move(P)); +} + bool TestNode::dns_resolve_start(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt blkid, std::string domain, int cat, int mode) { if (domain.size() > 1023) { @@ -1293,16 +1538,17 @@ bool TestNode::dns_resolve_start(ton::WorkchainId workchain, ton::StdSmcAddress workchain = ton::masterchainId; addr = dns_root_; } else { - auto P = td::PromiseCreator::lambda([this, blkid, domain, cat, mode](td::Result R) { - if (R.is_error()) { - LOG(ERROR) << "cannot obtain root dns address from configuration: " << R.move_as_error(); - } else if (dns_root_queried_) { - dns_resolve_start(ton::masterchainId, dns_root_, blkid, domain, cat, mode); - } else { - LOG(ERROR) << "cannot obtain root dns address from configuration parameter #4"; - } - }); - return get_config_params(mc_last_id_, std::move(P), 0x5000, "", {4}); + auto P = + td::PromiseCreator::lambda([this, blkid, domain, cat, mode](td::Result> R) { + if (R.is_error()) { + LOG(ERROR) << "cannot obtain root dns address from configuration: " << R.move_as_error(); + } else if (dns_root_queried_) { + dns_resolve_start(ton::masterchainId, dns_root_, blkid, domain, cat, mode); + } else { + LOG(ERROR) << "cannot obtain root dns address from configuration parameter #4"; + } + }); + return get_config_params(mc_last_id_, std::move(P), 0x3000, "", {4}); } } return dns_resolve_send(workchain, addr, blkid, domain, qdomain, cat, mode); @@ -2185,107 +2431,138 @@ void TestNode::got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::Bu show_new_blkids(); } -bool TestNode::get_config_params(ton::BlockIdExt blkid, td::Promise do_after, int mode, std::string filename, - std::vector params) { +bool TestNode::parse_get_config_params(ton::BlockIdExt blkid, int mode, std::string filename, std::vector params) { if (mode < 0) { - mode = 0x8000; + mode = 0x80000; } - if (!(mode & 0x9000) && !seekeoln()) { + if (!(mode & 0x81000) && !seekeoln()) { mode |= 0x1000; while (!seekeoln()) { int x; if (!convert_int32(get_word(), x)) { - do_after.set_error(td::Status::Error("integer configuration parameter id expected")); return set_error("integer configuration parameter id expected"); } params.push_back(x); } } if (!(ready_ && !client_.empty())) { - do_after.set_error(td::Status::Error("integer configuration parameter id expected")); return set_error("server connection not ready"); } if (!blkid.is_masterchain_ext()) { - do_after.set_error(td::Status::Error("integer configuration parameter id expected")); return set_error("only masterchain blocks contain configuration"); } if (blkid == mc_last_id_) { mode |= 0x2000; } + return get_config_params(blkid, trivial_promise_of>(), mode, filename, + std::move(params)); +} + +bool TestNode::get_config_params(ton::BlockIdExt blkid, td::Promise> promise, int mode, + std::string filename, std::vector params) { + return get_config_params_ext(blkid, promise.wrap([](ConfigInfo&& info) { return std::move(info.config); }), + mode | 0x10000, filename, params); +} + +bool TestNode::get_config_params_ext(ton::BlockIdExt blkid, td::Promise promise, int mode, + std::string filename, std::vector params) { + if (!(ready_ && !client_.empty())) { + promise.set_error(td::Status::Error("server connection not ready")); + return false; + } + if (!blkid.is_masterchain_ext()) { + promise.set_error(td::Status::Error("masterchain reference block expected")); + return false; + } + if (blkid == mc_last_id_) { + mode |= 0x2000; + } auto params_copy = params; - auto b = (mode & 0x1000) - ? ton::serialize_tl_object(ton::create_tl_object( - 0, ton::create_tl_lite_block_id(blkid), std::move(params_copy)), - true) - : ton::serialize_tl_object(ton::create_tl_object( - 0, ton::create_tl_lite_block_id(blkid)), - true); + auto b = (mode & 0x1000) ? ton::serialize_tl_object( + ton::create_tl_object( + mode & 0x8fff, ton::create_tl_lite_block_id(blkid), std::move(params_copy)), + true) + : ton::serialize_tl_object(ton::create_tl_object( + mode & 0x8fff, ton::create_tl_lite_block_id(blkid)), + true); LOG(INFO) << "requesting " << params.size() << " configuration parameters with respect to masterchain block " << blkid.to_str(); return envelope_send_query(std::move(b), [Self = actor_id(this), mode, filename, blkid, params = std::move(params), - do_after = std::move(do_after)](td::Result R) mutable { - if (R.is_error()) { - do_after.set_error(R.move_as_error()); - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getConfigParams"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, ton::create_block_id(f->id_), - std::move(f->state_proof_), std::move(f->config_proof_), mode, filename, - std::move(params), std::move(do_after)); - } + promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), + std::move(R), std::move(promise)); }); } -void TestNode::got_config_params(ton::BlockIdExt req_blkid, ton::BlockIdExt blkid, td::BufferSlice state_proof, - td::BufferSlice cfg_proof, int mode, std::string filename, std::vector params, - td::Promise do_after) { +void TestNode::got_config_params(ton::BlockIdExt req_blkid, int mode, std::string filename, std::vector params, + td::Result R, td::Promise promise) { + TRY_RESULT_PROMISE(promise, res, std::move(R)); + TRY_RESULT_PROMISE_PREFIX(promise, f, + ton::fetch_tl_object(std::move(res), true), + "cannot parse answer to liteServer.getConfigParams"); + auto blkid = ton::create_block_id(f->id_); LOG(INFO) << "got configuration parameters"; if (!blkid.is_masterchain_ext()) { - LOG(ERROR) << "reference block " << blkid.to_str() << " for the configuration is not a valid masterchain block"; + promise.set_error(td::Status::Error("reference block "s + blkid.to_str() + + " for the configuration is not a valid masterchain block")); return; } - if (blkid != req_blkid) { - LOG(ERROR) << "got configuration parameters with respect to block " << blkid.to_str() << " instead of " - << req_blkid.to_str(); - return; - } - auto R = block::check_extract_state_proof(blkid, state_proof.as_slice(), cfg_proof.as_slice()); - if (R.is_error()) { - LOG(ERROR) << "masterchain state proof for " << blkid.to_str() << " is invalid : " << R.move_as_error().to_string(); + bool from_key = (mode & 0x8000); + if (blkid.seqno() > req_blkid.seqno() || (!from_key && blkid != req_blkid)) { + promise.set_error(td::Status::Error("got configuration parameters with respect to block "s + blkid.to_str() + + " instead of " + req_blkid.to_str())); return; } try { - auto res = block::Config::extract_from_state(R.move_as_ok(), mode >= 0 ? mode & 0xfff : 0); - if (res.is_error()) { - LOG(ERROR) << "cannot unpack configuration: " << res.move_as_error().to_string(); - return; + Ref state, block, state_proof, config_proof; + if (!(mode & 0x10000) && !from_key) { + TRY_RESULT_PROMISE_PREFIX_ASSIGN(promise, state_proof, vm::std_boc_deserialize(f->state_proof_.as_slice()), + "cannot deserialize state proof :"); } - auto config = res.move_as_ok(); - if (mode & 0x8000) { - auto F = vm::std_boc_serialize(config->get_root_cell(), 2); - if (F.is_error()) { - LOG(ERROR) << "cannot serialize configuration: " << F.move_as_error().to_string(); - return; - } - auto size = F.ok().size(); - auto W = td::write_file(filename, F.move_as_ok()); - if (W.is_error()) { - LOG(ERROR) << "cannot save file `" << filename << "` : " << W.move_as_error().to_string(); + if (!(mode & 0x10000) || from_key) { + TRY_RESULT_PROMISE_PREFIX_ASSIGN(promise, config_proof, vm::std_boc_deserialize(f->config_proof_.as_slice()), + "cannot deserialize config proof :"); + } + if (!from_key) { + TRY_RESULT_PROMISE_PREFIX_ASSIGN( + promise, state, + block::check_extract_state_proof(blkid, f->state_proof_.as_slice(), f->config_proof_.as_slice()), + PSLICE() << "masterchain state proof for " << blkid.to_str() << " is invalid :"); + } else { + block = vm::MerkleProof::virtualize(config_proof, 1); + if (block.is_null()) { + promise.set_error( + td::Status::Error("cannot virtualize configuration proof constructed from key block "s + blkid.to_str())); return; } + //TRY_STATUS_PROMISE_PREFIX(promise, block::check_block_header_proof(block, blkid), + // PSLICE() << "incorrect header for key block " << blkid.to_str()); + } + TRY_RESULT_PROMISE_PREFIX(promise, config, + from_key ? block::Config::extract_from_key_block(block, mode & 0xfff) + : block::Config::extract_from_state(state, mode & 0xfff), + "cannot unpack configuration:"); + ConfigInfo cinfo{std::move(config), std::move(state_proof), std::move(config_proof)}; + if (mode & 0x80000) { + TRY_RESULT_PROMISE_PREFIX(promise, boc, vm::std_boc_serialize(cinfo.config->get_root_cell(), 2), + "cannot serialize configuration:"); + auto size = boc.size(); + TRY_STATUS_PROMISE_PREFIX(promise, td::write_file(filename, std::move(boc)), + PSLICE() << "cannot save file `" << filename << "` :"); td::TerminalIO::out() << "saved configuration dictionary into file `" << filename << "` (" << size << " bytes written)" << std::endl; + promise.set_result(std::move(cinfo)); + return; + } + if (mode & 0x4000) { + promise.set_result(std::move(cinfo)); return; } auto out = td::TerminalIO::out(); if (mode & 0x1000) { for (int i : params) { out << "ConfigParam(" << i << ") = "; - auto value = config->get_config_param(i); + auto value = cinfo.config->get_config_param(i); if (value.is_null()) { out << "(null)\n"; } else { @@ -2296,13 +2573,13 @@ void TestNode::got_config_params(ton::BlockIdExt req_blkid, ton::BlockIdExt blki } vm::load_cell_slice(value).print_rec(print_limit_, os); out << os.str() << std::endl; - if (i == 4 && (mode & 0x2000)) { - register_config_param4(value); + if (mode & 0x2000) { + register_config_param(i, value); } } } } else { - config->foreach_config_param([this, &out, mode](int i, Ref value) { + cinfo.config->foreach_config_param([this, &out, mode](int i, Ref value) { out << "ConfigParam(" << i << ") = "; if (value.is_null()) { out << "(null)\n"; @@ -2314,19 +2591,31 @@ void TestNode::got_config_params(ton::BlockIdExt req_blkid, ton::BlockIdExt blki } vm::load_cell_slice(value).print_rec(print_limit_, os); out << os.str() << std::endl; - if (i == 4 && (mode & 0x2000)) { - register_config_param4(value); + if (mode & 0x2000) { + register_config_param(i, value); } } return true; }); } + promise.set_result(std::move(cinfo)); } catch (vm::VmError& err) { - LOG(ERROR) << "error while traversing configuration: " << err.get_msg(); + promise.set_error(err.as_status("error while traversing configuration: ")); + return; } catch (vm::VmVirtError& err) { - LOG(ERROR) << "virtualization error while traversing configuration: " << err.get_msg(); + promise.set_error(err.as_status("virtualization error while traversing configuration: ")); + return; + } +} + +bool TestNode::register_config_param(int idx, Ref value) { + if (idx == 1) { + return register_config_param1(std::move(value)); + } else if (idx == 4) { + return register_config_param4(std::move(value)); + } else { + return true; } - do_after.set_result(td::Unit()); } bool TestNode::register_config_param4(Ref value) { @@ -2347,6 +2636,24 @@ bool TestNode::register_config_param4(Ref value) { } } +bool TestNode::register_config_param1(Ref value) { + if (value.is_null()) { + return false; + } + vm::CellSlice cs{vm::NoVmOrd(), std::move(value)}; + ton::StdSmcAddress addr; + if (cs.size_ext() == 256 && cs.fetch_bits_to(addr)) { + elect_addr_queried_ = true; + if (elect_addr_ != addr) { + elect_addr_ = addr; + LOG(INFO) << "elector smart contract address set to -1:" << addr.to_hex(); + } + return true; + } else { + return false; + } +} + bool TestNode::get_block(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got block download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( @@ -2517,61 +2824,97 @@ void TestNode::got_state(ton::BlockIdExt blkid, ton::RootHash root_hash, ton::Fi show_new_blkids(); } -bool TestNode::get_block_header(ton::BlockIdExt blkid, int mode) { +bool TestNode::get_show_block_header(ton::BlockIdExt blkid, int mode) { + return get_block_header(blkid, mode, [this, blkid](td::Result R) { + if (R.is_error()) { + LOG(ERROR) << "unable to fetch block header: " << R.move_as_error(); + } else { + auto res = R.move_as_ok(); + show_block_header(res.blk_id, res.virt_blk_root, res.mode); + show_new_blkids(); + } + }); +} + +bool TestNode::get_block_header(ton::BlockIdExt blkid, int mode, td::Promise promise) { LOG(INFO) << "got block header request for " << blkid.to_str() << " with mode " << mode; auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid), mode), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block header for " << blkid.to_str() - << " from server : " << res.move_as_error().to_string(); - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getBlockHeader : " << res.move_as_error().to_string(); - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained block header for " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); - } - td::actor::send_closure_later(Self, &TestNode::got_block_header, blk_id, std::move(f->header_proof_), f->mode_); - } - } - }); - return false; + return envelope_send_query( + std::move(b), [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { + TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), + PSLICE() << "cannot obtain block header for " << blkid.to_str() << " from server :"); + got_block_header_raw(std::move(res), std::move(promise), blkid); + }); } -bool TestNode::lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg) { +bool TestNode::lookup_show_block(ton::ShardIdFull shard, int mode, td::uint64 arg) { + return lookup_block(shard, mode, arg, [this](td::Result R) { + if (R.is_error()) { + LOG(ERROR) << "unable to look up block: " << R.move_as_error(); + } else { + auto res = R.move_as_ok(); + show_block_header(res.blk_id, res.virt_blk_root, res.mode); + show_new_blkids(); + } + }); +} + +bool TestNode::lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg, + td::Promise promise) { ton::BlockId id{shard, mode & 1 ? (td::uint32)arg : 0}; LOG(INFO) << "got block lookup request for " << id.to_str() << " with mode " << mode << " and argument " << arg; auto b = ton::serialize_tl_object(ton::create_tl_object( mode, ton::create_tl_lite_block_id_simple(id), arg, (td::uint32)arg), true); return envelope_send_query( - std::move(b), [Self = actor_id(this), id, mode, arg](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot look up block header for " << id.to_str() << " with mode " << mode << " and argument " - << arg << " from server : " << res.move_as_error().to_string(); - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.lookupBlock : " << res.move_as_error().to_string(); - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained block header for " << blk_id.to_str() << " from server"; - td::actor::send_closure_later(Self, &TestNode::got_block_header, blk_id, std::move(f->header_proof_), - f->mode_); - } - } + std::move(b), [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { + TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), + PSLICE() << "cannot look up block header for " << id.to_str() << " with mode " << mode + << " and argument " << arg << " from server :"); + got_block_header_raw(std::move(res), std::move(promise)); }); } +void TestNode::got_block_header_raw(td::BufferSlice res, td::Promise promise, + ton::BlockIdExt req_blkid) { + TRY_RESULT_PROMISE_PREFIX(promise, f, + ton::fetch_tl_object(std::move(res), true), + "cannot parse answer to liteServer.lookupBlock :"); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained block header for " << blk_id.to_str() << " from server (" << f->header_proof_.size() + << " data bytes)"; + if (req_blkid.is_valid() && blk_id != req_blkid) { + promise.set_error(td::Status::Error(PSLICE() << "block id mismatch: expected data for block " << req_blkid.to_str() + << ", obtained for " << blk_id.to_str())); + return; + } + TRY_RESULT_PROMISE_PREFIX(promise, root, vm::std_boc_deserialize(std::move(f->header_proof_)), + "cannot deserialize block header data :"); + bool ok = false; + td::Status E; + try { + auto virt_root = vm::MerkleProof::virtualize(root, 1); + if (virt_root.is_null()) { + promise.set_error(td::Status::Error(PSLICE() << "block header proof for block " << blk_id.to_str() + << " is not a valid Merkle proof")); + return; + } + ok = true; + promise.set_result(BlockHdrInfo{blk_id, std::move(root), std::move(virt_root), f->mode_}); + return; + } catch (vm::VmError& err) { + E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); + } catch (vm::VmVirtError& err) { + E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); + } + if (ok) { + LOG(ERROR) << std::move(E); + } else { + promise.set_error(std::move(E)); + } +} + bool TestNode::show_block_header(ton::BlockIdExt blkid, Ref root, int mode) { ton::RootHash vhash{root->get_hash().bits()}; if (vhash != blkid.root_hash) { @@ -2721,10 +3064,10 @@ void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod register_blkid(chain->key_blkid); } register_blkid(chain->to); - auto now = static_cast(td::Clocks::system()); - if (!(mode & 1) || (chain->last_utime > now - 3600)) { + auto time = now(); + if (!(mode & 1) || (chain->last_utime > time - 3600)) { td::TerminalIO::out() << "last block in chain was generated at " << chain->last_utime << " (" - << now - chain->last_utime << " seconds ago)\n"; + << time - chain->last_utime << " seconds ago)\n"; } show_new_blkids(); } @@ -2740,93 +3083,959 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c if (!(mode & 1)) { start_after.set_zero(); } - auto b = ton::serialize_tl_object(ton::create_tl_object( - mode, ton::create_tl_lite_block_id(blkid), req_count, start_after, min_utime), - true); - LOG(INFO) << "requesting up to " << req_count << " block creator stats records with respect to masterchain block " - << blkid.to_str() << " starting from validator public key " << start_after.to_hex() << " created after " - << min_utime << " (mode=" << mode << ")"; - return envelope_send_query(std::move(b), [Self = actor_id(this), mode, blkid, req_count, start_after, - min_utime](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getValidatorStats"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_creator_stats, blkid, ton::create_block_id(f->id_), mode, - f->mode_, start_after, min_utime, std::move(f->state_proof_), - std::move(f->data_proof_), f->count_, req_count, f->complete_); - } - }); + auto osp = std::make_unique(); + auto& os = *osp; + return get_creator_stats( + blkid, mode, req_count, start_after, min_utime, + [min_utime, &os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt) -> bool { + os << key.to_hex() << " mc_cnt:" << mc_cnt << " shard_cnt:" << shard_cnt << std::endl; + return true; + }, + td::PromiseCreator::lambda([os = std::move(osp)](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << "error obtaining creator stats: " << res.move_as_error(); + } else { + if (res.ok().is_zero()) { + *os << "(complete)" << std::endl; + } else { + *os << "(incomplete, repeat query from " << res.move_as_ok().to_hex() << " )" << std::endl; + } + td::TerminalIO::out() << os->str(); + } + })); } -void TestNode::got_creator_stats(ton::BlockIdExt req_blkid, ton::BlockIdExt blkid, int req_mode, int mode, - td::Bits256 start_after, ton::UnixTime min_utime, td::BufferSlice state_proof, - td::BufferSlice data_proof, int count, int req_count, bool complete) { +bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_count, ton::Bits256 start_after, + ton::UnixTime min_utime, TestNode::creator_stats_func_t func, + td::Promise promise) { + return get_creator_stats(blkid, req_count, min_utime, std::move(func), + std::make_unique(mode | 0x10000, start_after), + promise.wrap([](auto&& p) { return p->last_key; })); +} + +bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton::UnixTime min_utime, + TestNode::creator_stats_func_t func, std::unique_ptr state, + td::Promise> promise) { + if (!(ready_ && !client_.empty())) { + promise.set_error(td::Status::Error("server connection not ready")); + return false; + } + if (!state) { + promise.set_error(td::Status::Error("null CreatorStatsRes")); + return false; + } + if (!blkid.is_masterchain_ext()) { + promise.set_error(td::Status::Error("only masterchain blocks contain block creator statistics")); + return false; + } + if (!(state->mode & 1)) { + state->last_key.set_zero(); + } + auto b = ton::serialize_tl_object( + ton::create_tl_object( + state->mode & 0xff, ton::create_tl_lite_block_id(blkid), req_count, state->last_key, min_utime), + true); + LOG(INFO) << "requesting up to " << req_count << " block creator stats records with respect to masterchain block " + << blkid.to_str() << " starting from validator public key " << state->last_key.to_hex() << " created after " + << min_utime << " (mode=" << state->mode << ")"; + return envelope_send_query( + std::move(b), [this, blkid, req_count, state = std::move(state), min_utime, func = std::move(func), + promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, res, std::move(R)); + TRY_RESULT_PROMISE_PREFIX(promise, f, + ton::fetch_tl_object(std::move(res), true), + "cannot parse answer to liteServer.getValidatorStats"); + got_creator_stats(blkid, ton::create_block_id(f->id_), f->mode_, min_utime, std::move(f->state_proof_), + std::move(f->data_proof_), f->count_, req_count, f->complete_, std::move(func), + std::move(state), std::move(promise)); + }); +} + +void TestNode::got_creator_stats(ton::BlockIdExt req_blkid, ton::BlockIdExt blkid, int mode, ton::UnixTime min_utime, + td::BufferSlice state_proof, td::BufferSlice data_proof, int count, int req_count, + bool complete, TestNode::creator_stats_func_t func, + std::unique_ptr status, + td::Promise> promise) { LOG(INFO) << "got answer to getValidatorStats query: " << count << " records out of " << req_count << ", " << (complete ? "complete" : "incomplete"); if (!blkid.is_masterchain_ext()) { - LOG(ERROR) << "reference block " << blkid.to_str() - << " for block creator statistics is not a valid masterchain block"; + promise.set_error(td::Status::Error(PSLICE() << "reference block " << blkid.to_str() + << " for block creator statistics is not a valid masterchain block")); return; } if (count > req_count) { - LOG(ERROR) << "obtained " << count << " answers to getValidatorStats query, but only " << req_count - << " were requested"; + promise.set_error(td::Status::Error(PSLICE() + << "obtained " << count << " answers to getValidatorStats query, but only " + << req_count << " were requested")); return; } if (blkid != req_blkid) { - LOG(ERROR) << "answer to getValidatorStats refers to masterchain block " << blkid.to_str() - << " different from requested " << req_blkid.to_str(); + promise.set_error(td::Status::Error(PSLICE() + << "answer to getValidatorStats refers to masterchain block " << blkid.to_str() + << " different from requested " << req_blkid.to_str())); return; } - auto R = block::check_extract_state_proof(blkid, state_proof.as_slice(), data_proof.as_slice()); - if (R.is_error()) { - LOG(ERROR) << "masterchain state proof for " << blkid.to_str() << " is invalid : " << R.move_as_error().to_string(); - return; + TRY_RESULT_PROMISE_PREFIX(promise, state, + block::check_extract_state_proof(blkid, state_proof.as_slice(), data_proof.as_slice()), + PSLICE() << "masterchain state proof for " << blkid.to_str() << " is invalid :"); + if (!(mode & 0x10000)) { + if (status->state_proof.is_null()) { + TRY_RESULT_PROMISE_PREFIX( + promise, state_root, vm::std_boc_deserialize(state_proof.as_slice()), + PSLICE() << "cannot deserialize masterchain state proof for " << blkid.to_str() << ": "); + status->state_proof = std::move(state_root); + } + TRY_RESULT_PROMISE_PREFIX( + promise, data_root, vm::std_boc_deserialize(data_proof.as_slice()), + PSLICE() << "cannot deserialize masterchain creators data proof for " << blkid.to_str() << ": "); + if (status->data_proof.is_null()) { + status->data_proof = std::move(data_root); + } else { + TRY_RESULT_PROMISE_PREFIX(promise, data_proof2, + vm::MerkleProof::combine_fast_status(status->data_proof, std::move(data_root)), + "cannot combine Merkle proofs for creator data"); + status->data_proof = std::move(data_proof2); + } } bool allow_eq = (mode & 3) != 1; - ton::Bits256 key{start_after}; + ton::Bits256 key{status->last_key}; std::ostringstream os; try { - auto dict = block::get_block_create_stats_dict(R.move_as_ok()); + auto dict = block::get_block_create_stats_dict(std::move(state)); if (!dict) { - LOG(ERROR) << "cannot extract BlockCreateStats from mc state"; + promise.set_error(td::Status::Error("cannot extract BlockCreateStats from mc state")); return; } for (int i = 0; i < count + (int)complete; i++) { auto v = dict->lookup_nearest_key(key, true, allow_eq); if (v.is_null()) { if (i != count) { - LOG(ERROR) << "could fetch only " << i << " CreatorStats entries out of " << count - << " declared in answer to getValidatorStats"; + promise.set_error(td::Status::Error(PSLICE() << "could fetch only " << i << " CreatorStats entries out of " + << count << " declared in answer to getValidatorStats")); return; } break; } block::DiscountedCounter mc_cnt, shard_cnt; if (!block::unpack_CreatorStats(std::move(v), mc_cnt, shard_cnt)) { - LOG(ERROR) << "invalid CreatorStats record with key " << key.to_hex(); + promise.set_error(td::Status::Error(PSLICE() << "invalid CreatorStats record with key " << key.to_hex())); return; } if (mc_cnt.modified_since(min_utime) || shard_cnt.modified_since(min_utime)) { - os << key.to_hex() << " mc_cnt:" << mc_cnt << " shard_cnt:" << shard_cnt << std::endl; + func(key, mc_cnt, shard_cnt); } allow_eq = false; } if (complete) { - os << "(complete)" << std::endl; + status->last_key.set_zero(); + status->complete = true; + promise.set_result(std::move(status)); + } else if (!(status->mode & 0x100)) { + status->last_key = key; + promise.set_result(std::move(status)); } else { - os << "(incomplete, repeat query from " << key.to_hex() << " )" << std::endl; + // incomplete, send new query to fetch next entries + status->last_key = key; + status->mode |= 1; + get_creator_stats(blkid, req_count, min_utime, std::move(func), std::move(status), std::move(promise)); } - td::TerminalIO::out() << os.str(); } catch (vm::VmError& err) { - LOG(ERROR) << "error while traversing block creator stats: " << err.get_msg(); + promise.set_error(err.as_status("error while traversing block creator stats:")); } catch (vm::VmVirtError& err) { - LOG(ERROR) << "virtualization error while traversing block creator stats: " << err.get_msg(); + promise.set_error(err.as_status("virtualization error while traversing block creator stats:")); + } +} + +bool TestNode::check_validator_load(int start_time, int end_time, int mode, std::string file_pfx) { + int time = now(); + if (start_time <= 0) { + start_time += time; + } + if (end_time <= 0) { + end_time += time; + } + if (start_time >= end_time) { + return set_error("end time must be later than start time"); + } + LOG(INFO) << "requesting masterchain blocks corresponding to unixtime " << start_time << " and " << end_time; + auto P = td::split_promise([this, mode, file_pfx](td::Result> R) { + if (R.is_error()) { + LOG(ERROR) << "cannot obtain block info: " << R.move_as_error(); + return; + } + auto res = R.move_as_ok(); + continue_check_validator_load(res.first.blk_id, res.first.proof, res.second.blk_id, res.second.proof, mode, + file_pfx); + }); + lookup_block(ton::ShardIdFull(ton::masterchainId), 4, start_time, std::move(P.first)); + return lookup_block(ton::ShardIdFull(ton::masterchainId), 4, end_time, std::move(P.second)); +} + +void TestNode::continue_check_validator_load(ton::BlockIdExt blkid1, Ref root1, ton::BlockIdExt blkid2, + Ref root2, int mode, std::string file_pfx) { + LOG(INFO) << "continue_check_validator_load for blocks " << blkid1.to_str() << " and " << blkid2.to_str() + << " : requesting configuration parameter #34"; + auto P = td::split_promise( + [this, blkid1, root1, blkid2, root2, mode, file_pfx](td::Result> R) mutable { + if (R.is_error()) { + LOG(ERROR) << "cannot obtain configuration parameter #34 : " << R.move_as_error(); + return; + } + auto res = R.move_as_ok(); + root1 = vm::MerkleProof::combine_fast(std::move(root1), std::move(res.first.state_proof)); + root2 = vm::MerkleProof::combine_fast(std::move(root2), std::move(res.second.state_proof)); + if (root1.is_null() || root2.is_null()) { + LOG(ERROR) << "cannot merge block header proof with block state proof"; + return; + } + auto info1 = std::make_unique(blkid1, std::move(root1), std::move(res.first.config_proof), + std::move(res.first.config)); + auto info2 = std::make_unique(blkid2, std::move(root2), std::move(res.second.config_proof), + std::move(res.second.config)); + continue_check_validator_load2(std::move(info1), std::move(info2), mode, file_pfx); + }); + get_config_params_ext(blkid1, std::move(P.first), 0x4000, "", {28, 34}); + get_config_params_ext(blkid2, std::move(P.second), 0x4000, "", {28, 34}); +} + +td::Status TestNode::ValidatorLoadInfo::unpack_vset() { + if (!config) { + return td::Status::Error("no configuration to unpack validator set"); + } + auto vset_root_c = config->get_config_param(34); + if (vset_root_c.is_null()) { + vset_hash.set_zero(); + return td::Status::Error("no configuration parameter 34 for block "s + blk_id.to_str()); + } + if (vset_root.not_null() && vset_root->get_hash() == vset_root_c->get_hash()) { + vset_root_c = vset_root; + } else { + vset_root = vset_root_c; + } + vset_hash = vset_root->get_hash().bits(); + TRY_RESULT_PREFIX_ASSIGN( + vset, block::Config::unpack_validator_set(vset_root), + PSLICE() << "cannot unpack validator set from configuration parameter 34 of block " << blk_id.to_str() << " :"); + valid_since = vset->utime_since; + vset_map = vset->compute_validator_map(); + return td::Status::OK(); +} + +bool TestNode::ValidatorLoadInfo::store_record(const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt) { + if (!(mc_cnt.is_valid() && shard_cnt.is_valid())) { + return false; + } + if (mc_cnt.total >= (1ULL << 60) || shard_cnt.total >= (1ULL << 60)) { + return false; + } + if (key.is_zero()) { + created_total.first = (td::int64)mc_cnt.total; + created_total.second = (td::int64)shard_cnt.total; + return true; + } + auto it = vset_map.find(key); + if (it == vset_map.end()) { + return false; + } + created.at(it->second) = std::make_pair(mc_cnt.total, shard_cnt.total); + return true; +} + +bool TestNode::load_creator_stats(std::unique_ptr load_to, + td::Promise> promise, bool need_proofs) { + if (!load_to) { + promise.set_error(td::Status::Error("no ValidatorLoadInfo")); + return false; + } + auto& info = *load_to; + info.created_total.first = info.created_total.second = 0; + info.created.resize(0); + info.created.resize(info.vset->total, std::make_pair(0, 0)); + ton::UnixTime min_utime = info.valid_since - 1000; + return get_creator_stats( + info.blk_id, 1000, min_utime, + [min_utime, &info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt) -> bool { + info.store_record(key, mc_cnt, shard_cnt); + return true; + }, + std::make_unique(need_proofs ? 0x100 : 0x10100), + td::PromiseCreator::lambda([load_to = std::move(load_to), promise = std::move(promise)]( + td::Result> R) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), "error obtaining creator stats:"); + if (!res->complete) { + promise.set_error(td::Status::Error("incomplete creator stats")); + return; + } + // merge + load_to->state_proof = + vm::MerkleProof::combine_fast(std::move(load_to->state_proof), std::move(res->state_proof)); + load_to->data_proof = vm::MerkleProof::combine_fast(std::move(load_to->data_proof), std::move(res->data_proof)); + promise.set_result(std::move(load_to)); + })); +} + +void TestNode::continue_check_validator_load2(std::unique_ptr info1, + std::unique_ptr info2, int mode, + std::string file_pfx) { + LOG(INFO) << "continue_check_validator_load2 for blocks " << info1->blk_id.to_str() << " and " + << info1->blk_id.to_str() << " : requesting block creators data"; + td::Status st = info1->unpack_vset(); + if (st.is_error()) { + LOG(ERROR) << "cannot unpack validator set from block " << info1->blk_id.to_str() << " :" << st.move_as_error(); + return; + } + st = info2->unpack_vset(); + if (st.is_error()) { + LOG(ERROR) << "cannot unpack validator set from block " << info2->blk_id.to_str() << " :" << st.move_as_error(); + return; + } + if (info1->vset_hash != info2->vset_hash || info1->valid_since != info2->valid_since) { + LOG(ERROR) << "blocks appear to have different validator sets"; + return; + } + LOG(INFO) << "validator sets valid since " << info1->valid_since; + auto P = td::split_promise( + [this, mode, + file_pfx](td::Result, std::unique_ptr>> R) { + if (R.is_error()) { + LOG(ERROR) << "cannot load block creation statistics : " << R.move_as_error(); + return; + } + auto res = R.move_as_ok(); + continue_check_validator_load3(std::move(res.first), std::move(res.second), mode, file_pfx); + }); + load_creator_stats(std::move(info1), std::move(P.first), true); + load_creator_stats(std::move(info2), std::move(P.second), true); +} + +// computes the probability of creating <= x masterchain blocks if the expected value is y +static double create_prob(int x, double y) { + if (x < 0 || y < 0) { + return .5; + } + if (x >= y) { + return .5; + } + if (x <= 20) { + // Poisson + double t = exp(-y), s = t; + for (int n = 1; n <= x; n++) { + s += t = (t * y) / n; + } + return s; + } + // normal approximation + double z = (x - y) / sqrt(2. * y); + return (1. + erf(z)) / 2; +} + +static double shard_create_prob(int x, double y, double chunk_size) { + if (x < 0 || y < 0) { + return .5; + } + if (x >= y) { + return .5; + } + double y0 = y / chunk_size; // expected chunks + if (!x) { + return y0 > 100 ? 0 : exp(-y0); // Poisson approximation for having participated in zero chunks + } + // at least ten chunks, normal approximation + double z = (x - y) / sqrt(2. * y * chunk_size); + return (1. + erf(z)) / 2; +} + +void TestNode::continue_check_validator_load3(std::unique_ptr info1, + std::unique_ptr info2, int mode, + std::string file_pfx) { + LOG(INFO) << "continue_check_validator_load3 for blocks " << info1->blk_id.to_str() << " and " + << info1->blk_id.to_str() << " with mode=" << mode << " and file prefix `" << file_pfx + << "`: comparing block creators data"; + if (info1->created_total.first <= 0 || info2->created_total.first <= 0) { + LOG(ERROR) << "no total created blocks statistics"; + return; + } + td::TerminalIO::out() << "total: (" << info1->created_total.first << "," << info1->created_total.second << ") -> (" + << info2->created_total.first << "," << info2->created_total.second << ")\n"; + auto x = info2->created_total.first - info1->created_total.first; + auto y = info2->created_total.second - info1->created_total.second; + td::int64 xs = 0, ys = 0; + if (x <= 0 || y < 0 || (x | y) >= (1u << 31)) { + LOG(ERROR) << "impossible situation: zero or no blocks created: " << x << " masterchain blocks, " << y + << " shardchain blocks"; + return; + } + std::pair created_total{(int)x, (int)y}; + int count = info1->vset->total; + CHECK(info2->vset->total == count); + CHECK((int)info1->created.size() == count); + CHECK((int)info2->created.size() == count); + std::vector> d; + d.reserve(count); + for (int i = 0; i < count; i++) { + auto x1 = info2->created[i].first - info1->created[i].first; + auto y1 = info2->created[i].second - info1->created[i].second; + if (x1 < 0 || y1 < 0 || (x1 | y1) >= (1u << 31)) { + LOG(ERROR) << "impossible situation: validator #i created a negative amount of blocks: " << x1 + << " masterchain blocks, " << y1 << " shardchain blocks"; + return; + } + xs += x1; + ys += y1; + d.emplace_back((int)x1, (int)y1); + td::TerminalIO::out() << "val #" << i << ": created (" << x1 << "," << y1 << ") ; was (" << info1->created[i].first + << "," << info1->created[i].second << ")\n"; + } + if (xs != x || ys != y) { + LOG(ERROR) << "cannot account for all blocks created: total is (" << x << "," << y + << "), but the sum for all validators is (" << xs << "," << ys << ")"; + return; + } + td::TerminalIO::out() << "total: (" << x << "," << y << ")\n"; + auto ccfg = block::Config::unpack_catchain_validators_config(info2->config->get_config_param(28)); + auto ccfg_old = block::Config::unpack_catchain_validators_config(info1->config->get_config_param(28)); + if (ccfg.shard_val_num != ccfg_old.shard_val_num || ccfg.shard_val_num <= 0) { + LOG(ERROR) << "shard validator group size changed from " << ccfg_old.shard_val_num << " to " << ccfg.shard_val_num + << ", or is not positive"; + return; + } + int shard_count = ccfg.shard_val_num, main_count = info2->vset->main; + if (info1->vset->main != main_count || main_count <= 0) { + LOG(ERROR) << "masterchain validator group size changed from " << info1->vset->main << " to " << main_count + << ", or is not positive"; + return; + } + int cnt = 0, cnt_ok = 0; + double chunk_size = ccfg.shard_val_lifetime / 3. / shard_count; + block::MtCarloComputeShare shard_share(shard_count, info2->vset->export_scaled_validator_weights()); + for (int i = 0; i < count; i++) { + int x1 = d[i].first, y1 = d[i].second; + double xe = (i < main_count ? (double)xs / main_count : 0); + double ye = shard_share[i] * (double)ys / shard_count; + td::Bits256 pk = info2->vset->list[i].pubkey.as_bits256(); + double p1 = create_prob(x1, .9 * xe), p2 = shard_create_prob(y1, .9 * ye, chunk_size); + td::TerminalIO::out() << "val #" << i << ": pubkey " << pk.to_hex() << ", blocks created (" << x1 << "," << y1 + << "), expected (" << xe << "," << ye << "), probabilities " << p1 << " and " << p2 << "\n"; + if (std::min(p1, p2) < .00001) { + LOG(ERROR) << "validator #" << i << " with pubkey " << pk.to_hex() + << " : serious misbehavior detected: created less than 90% of the expected amount of blocks with " + "probability 99.999% : created (" + << x1 << "," << y1 << "), expected (" << xe << "," << ye << ") masterchain/shardchain blocks\n"; + if (mode & 2) { + auto st = write_val_create_proof(*info1, *info2, i, true, file_pfx, ++cnt); + if (st.is_error()) { + LOG(ERROR) << "cannot create proof: " << st.move_as_error(); + } else { + cnt_ok++; + } + } + } else if (std::min(p1, p2) < .001) { + LOG(ERROR) << "validator #" << i << " with pubkey " << pk.to_hex() + << " : moderate misbehavior detected: created less than 90% of the expected amount of blocks with " + "probability 99.9% : created (" + << x1 << "," << y1 << "), expected (" << xe << "," << ye << ") masterchain/shardchain blocks\n"; + if ((mode & 3) == 2) { + auto st = write_val_create_proof(*info1, *info2, i, false, file_pfx, ++cnt); + if (st.is_error()) { + LOG(ERROR) << "cannot create proof: " << st.move_as_error(); + } else { + cnt_ok++; + } + } + } + } + if (cnt > 0) { + LOG(INFO) << cnt_ok << " out of " << cnt << " proofs written to " << file_pfx << "-*.boc"; + } +} + +bool compute_punishment(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part) { + if (interval <= 1000) { + return false; // no punishments for less than 1000 seconds + } + if (severe) { + fine = td::make_refint(2500 * 1000000000LL); // GR$2500 + fine_part = (1 << 30); // 1/4 of stake + } else { + fine = td::make_refint(1000 * 1000000000LL); // GR$1000 + fine_part = (1 << 28); // 1/16 of stake + } + if (interval >= 80000) { + return true; + } + if (interval >= 20000) { + fine >>= 2; + fine_part >>= 2; + return true; + } + fine >>= 4; + fine_part >>= 4; + return true; +} + +bool check_punishment(int interval, bool severe, td::RefInt256 fine, unsigned fine_part) { + td::RefInt256 computed_fine; + unsigned computed_fine_part; + return compute_punishment(interval, severe, computed_fine, computed_fine_part) && + std::llabs((long long)fine_part - (long long)computed_fine_part) <= + (std::max(fine_part, computed_fine_part) >> 3) && + fine * 7 <= computed_fine * 8 && computed_fine * 7 <= fine * 8; +} + +td::Status TestNode::write_val_create_proof(TestNode::ValidatorLoadInfo& info1, TestNode::ValidatorLoadInfo& info2, + int idx, bool severe, std::string file_pfx, int cnt) { + std::string filename = PSTRING() << file_pfx << '-' << cnt << ".boc"; + if (!info1.has_data()) { + return td::Status::Error("first block information is incomplete"); + } + if (!info2.has_data()) { + return td::Status::Error("second block information is incomplete"); + } + LOG(INFO) << "creating proof file " << filename; + TRY_STATUS(info1.check_header_proof(&info1.block_created_at, &info1.end_lt)); + TRY_STATUS(info2.check_header_proof(&info2.block_created_at, &info2.end_lt)); + td::Bits256 val_pk1, val_pk2; + TRY_RESULT(prod1, info1.build_producer_info(idx, &val_pk1)); + TRY_RESULT(prod2, info2.build_producer_info(idx, &val_pk2)); + if (val_pk1 != val_pk2) { + return td::Status::Error("validator public key mismatch"); + } + int interval = (int)(info2.block_created_at - info1.block_created_at); + if (interval <= 0) { + return td::Status::Error("non-positive time interval"); + } + int severity = (severe ? 2 : 1); + td::RefInt256 fine = td::make_refint(1000000000); + unsigned fine_part = 0xffffffff / 16; // 1/16 + if (!compute_punishment(interval, severe, fine, fine_part)) { + return td::Status::Error("cannot compute adequate punishment"); + } + Ref cpl_descr, complaint; + vm::CellBuilder cb; + // no_blk_gen_diff prod_info_old:^ProducerInfo prod_info_new:^ProducerInfo = ComplaintDescr + if (!(block::gen::t_ComplaintDescr.cell_pack_no_blk_gen_diff(cpl_descr, prod1, prod2) && + cb.store_long_bool(0xbc, 8) // validator_complaint#bc + && cb.store_bits_bool(val_pk1) // validator_pubkey:uint256 + && cb.store_ref_bool(cpl_descr) // description:^ComplaintDescr + && cb.store_long_bool(now(), 32) // created_at:uint32 + && cb.store_long_bool(severity, 8) // severity:uint8 + && cb.store_zeroes_bool(256) // reward_addr:uint256 + && cb.store_zeroes_bool(4) // paid:Grams + && block::tlb::t_Grams.store_integer_ref(cb, std::move(fine)) // suggested_fine:Grams + && cb.store_long_bool(fine_part, 32) // suggested_fine_part:uint32 + && cb.finalize_to(complaint))) { + return td::Status::Error("cannot serialize ValidatorComplaint"); + } + if (verbosity >= 5) { + std::ostringstream os; + os << "complaint: "; + block::gen::t_ValidatorComplaint.print_ref(print_limit_, os, complaint); + td::TerminalIO::out() << os.str() << std::endl; + } + if (!block::gen::t_ComplaintDescr.validate_ref(cpl_descr)) { + return td::Status::Error("created an invalid ComplaintDescr"); + } + if (!block::gen::t_ValidatorComplaint.validate_ref(complaint)) { + return td::Status::Error("created an invalid ValidatorComplaint"); + } + TRY_RESULT_PREFIX(boc, vm::std_boc_serialize(complaint, 2), "cannot create boc:"); + auto size = boc.size(); + TRY_STATUS_PREFIX(td::write_file(filename, std::move(boc)), PSLICE() << "cannot save file `" << filename << "` :"); + td::TerminalIO::out() << "saved validator misbehavior proof into file `" << filename << "` (" << size + << " bytes written)" << std::endl; + td::TerminalIO::out() << "COMPLAINT_SAVED\t" << info1.vset_hash.to_hex() << "\t" << complaint->get_hash().to_hex() + << "\t" << filename << std::endl; + return td::Status::OK(); +} + +td::Status TestNode::ValidatorLoadInfo::check_header_proof(ton::UnixTime* save_utime, ton::LogicalTime* save_lt) const { + auto state_virt_root = vm::MerkleProof::virtualize(std::move(data_proof), 1); + if (state_virt_root.is_null()) { + return td::Status::Error("account state proof is invalid"); + } + td::Bits256 state_hash = state_virt_root->get_hash().bits(); + TRY_STATUS(block::check_block_header_proof(vm::MerkleProof::virtualize(state_proof, 1), blk_id, &state_hash, true, + save_utime, save_lt)); + return td::Status::OK(); +} + +static bool visit(Ref cell); + +static bool visit(const vm::CellSlice& cs) { + auto cnt = cs.size_refs(); + bool res = true; + for (unsigned i = 0; i < cnt; i++) { + res &= visit(cs.prefetch_ref(i)); + } + return res; +} + +static bool visit(Ref cell) { + if (cell.is_null()) { + return true; + } + vm::CellSlice cs{vm::NoVm{}, std::move(cell)}; + return visit(cs); +} + +static bool visit(Ref cs_ref) { + return cs_ref.is_null() || visit(*cs_ref); +} + +td::Result> TestNode::ValidatorLoadInfo::build_proof(int idx, td::Bits256* save_pubkey) const { + try { + auto state_virt_root = vm::MerkleProof::virtualize(std::move(data_proof), 1); + if (state_virt_root.is_null()) { + return td::Status::Error("account state proof is invalid"); + } + vm::MerkleProofBuilder pb{std::move(state_virt_root)}; + TRY_RESULT(cfg, block::Config::extract_from_state(pb.root())); + visit(cfg->get_config_param(28)); + block::gen::ValidatorSet::Record_validators_ext rec; + if (!tlb::unpack_cell(cfg->get_config_param(34), rec)) { + return td::Status::Error("cannot unpack ValidatorSet"); + } + vm::Dictionary vdict{rec.list, 16}; + auto entry = vdict.lookup(td::BitArray<16>(idx)); + if (entry.is_null()) { + return td::Status::Error("validator entry not found"); + } + Ref pk; + block::gen::ValidatorDescr::Record_validator rec1; + block::gen::ValidatorDescr::Record_validator_addr rec2; + if (tlb::csr_unpack(entry, rec1)) { + pk = std::move(rec1.public_key); + } else if (tlb::csr_unpack(std::move(entry), rec2)) { + pk = std::move(rec2.public_key); + } else { + return td::Status::Error("cannot unpack ValidatorDescr"); + } + block::gen::SigPubKey::Record rec3; + if (!tlb::csr_unpack(std::move(pk), rec3)) { + return td::Status::Error("cannot unpack ed25519_pubkey"); + } + if (save_pubkey) { + *save_pubkey = rec3.pubkey; + } + visit(std::move(entry)); + auto dict = block::get_block_create_stats_dict(pb.root()); + if (!dict) { + return td::Status::Error("cannot extract BlockCreateStats from mc state"); + } + visit(dict->lookup(rec3.pubkey)); + visit(dict->lookup(td::Bits256::zero())); + return pb.extract_proof(); + } catch (vm::VmError& err) { + return err.as_status("cannot build proof: "); + } catch (vm::VmVirtError& err) { + return err.as_status("cannot build proof: "); + } +} + +td::Result> TestNode::ValidatorLoadInfo::build_producer_info(int idx, td::Bits256* save_pubkey) const { + TRY_RESULT(proof, build_proof(idx, save_pubkey)); + vm::CellBuilder cb; + Ref res; + if (!(cb.store_long_bool(0x34, 8) // prod_info#34 + && cb.store_long_bool(block_created_at, 32) // utime:uint32 + && block::tlb::t_ExtBlkRef.store(cb, blk_id, end_lt) // mc_blk_ref:ExtBlkRef + && cb.store_ref_bool(state_proof) // state_proof:^Cell + && cb.store_ref_bool(proof) // prod_proof:^Cell = ProducerInfo + && cb.finalize_to(res))) { + return td::Status::Error("cannot construct ProducerInfo"); + } + if (!block::gen::t_ProducerInfo.validate_ref(res)) { + return td::Status::Error("constructed ProducerInfo failed to pass automated validity checks"); + } + return std::move(res); +} + +td::Status TestNode::check_validator_load_proof(std::string filename, std::string vset_filename, + ton::Bits256 vset_hash) { + TRY_RESULT_PREFIX(data, td::read_file(filename), "cannot read proof file:"); + TRY_RESULT_PREFIX(root, vm::std_boc_deserialize(std::move(data)), + PSTRING() << "cannot deserialize boc from file `" << filename << "`:"); + Ref vset_root; + if (!vset_filename.empty()) { + TRY_RESULT_PREFIX(vdata, td::read_file(vset_filename), "cannot read validator set file:"); + TRY_RESULT_PREFIX_ASSIGN(vset_root, vm::std_boc_deserialize(std::move(vdata)), + PSLICE() << "cannot deserialize validator set boc from file `" << vset_filename << "`:"); + if (vset_hash.is_zero()) { + vset_hash = vset_root->get_hash().bits(); + } else if (vset_hash != vset_root->get_hash().bits()) { + return td::Status::Error("validator set hash mismatch: indicated "s + vset_hash.to_hex() + ", loaded from file " + + vset_root->get_hash().to_hex()); + } + } + if (verbosity >= 5) { + std::ostringstream os; + os << "complaint: "; + block::gen::t_ValidatorComplaint.print_ref(print_limit_, os, root); + td::TerminalIO::out() << os.str() << std::endl; + } + if (!block::gen::t_ValidatorComplaint.validate_ref(root)) { + return td::Status::Error("proof file does not contain a valid ValidatorComplaint"); + } + block::gen::ValidatorComplaint::Record rec; + if (!tlb::unpack_cell(root, rec)) { + return td::Status::Error("cannot unpack ValidatorComplaint"); + } + auto cs = vm::load_cell_slice(rec.description); + int tag = block::gen::t_ComplaintDescr.get_tag(cs); + if (tag < 0) { + return td::Status::Error("ComplaintDescr has an unknown tag"); + } + if (tag != block::gen::ComplaintDescr::no_blk_gen_diff) { + return td::Status::Error("can check only ComplaintDescr of type no_blk_gen_diff"); + } + block::gen::ComplaintDescr::Record_no_blk_gen_diff crec; + if (!tlb::unpack_exact(cs, crec)) { + return td::Status::Error("cannot unpack ComplaintDescr"); + } + TRY_RESULT_PREFIX(info1, ValidatorLoadInfo::preinit_from_producer_info(crec.prod_info_old), + "cannot unpack ProducerInfo in prod_info_old:") + TRY_RESULT_PREFIX(info2, ValidatorLoadInfo::preinit_from_producer_info(crec.prod_info_new), + "cannot unpack ProducerInfo in prod_info_new:") + if (info1->vset_hash != info2->vset_hash) { + return td::Status::Error("validator hash changed between the two blocks: "s + info1->vset_hash.to_hex() + " and " + + info2->vset_hash.to_hex()); + } + if (vset_hash.is_zero()) { + vset_hash = info1->vset_hash; + } else if (vset_hash != info1->vset_hash) { + return td::Status::Error("validator set hash mismatch: blocks have "s + info1->vset_hash.to_hex() + + ", actual value is " + vset_hash.to_hex()); + } + auto blkid2 = info2->blk_id; + if (vset_root.not_null()) { + info1->vset_root = info2->vset_root = std::move(vset_root); + set_error(continue_check_validator_load_proof(std::move(info1), std::move(info2), std::move(root))); + } else if (!get_config_params(blkid2, + [this, info1 = std::move(info1), info2 = std::move(info2), + root = std::move(root)](td::Result> res) mutable { + if (res.is_error()) { + LOG(ERROR) + << "cannot fetch configuration parameters from key block corresponding to " + << info2->blk_id.to_str() << " : " << res.move_as_error(); + } else { + auto vset_root = res.move_as_ok()->get_config_param(34); + if (vset_root.is_null()) { + LOG(ERROR) << "no configuration parameter #34 in key block corresponding to " + << info2->blk_id.to_str(); + } else if (info2->vset_hash != vset_root->get_hash().bits()) { + LOG(ERROR) << "validator hash set mismatch for block " << info2->blk_id.to_str(); + } else { + info1->vset_root = info2->vset_root = std::move(vset_root); + set_error(continue_check_validator_load_proof(std::move(info1), std::move(info2), + std::move(root))); + } + } + }, + 0xd000, "", {28, 34})) { + return td::Status::Error("cannot request configuration parameters from key block corresponding to "s + + blkid2.to_str()); + } + return td::Status::OK(); +} + +static void show_vote(td::Bits256 complaint_hash, bool outcome) { + td::TerminalIO::out() << "COMPLAINT_VOTE_FOR\t" << complaint_hash.to_hex() << "\t" << (outcome ? "YES" : "NO") + << std::endl; +} + +td::Status TestNode::continue_check_validator_load_proof(std::unique_ptr info1, + std::unique_ptr info2, Ref root) { + TRY_STATUS(info1->unpack_vset()); + TRY_STATUS(info2->unpack_vset()); + int interval = (int)(info2->block_created_at - info1->block_created_at); + if (interval <= 0) { + return td::Status::Error("non-positive time interval"); + } + block::gen::ValidatorComplaint::Record rec; + block::gen::ComplaintDescr::Record_no_blk_gen_diff crec; + if (!(tlb::unpack_cell(root, rec) && tlb::unpack_cell(rec.description, crec))) { + return td::Status::Error("cannot unpack ValidatorComplaint second time (?)"); + } + td::Bits256 val_pubkey = rec.validator_pubkey; + int val_idx = info1->vset->lookup_public_key(val_pubkey); + if (val_idx < 0) { + return td::Status::Error("validator with public key "s + val_pubkey.to_hex() + + " is not present in active validator set"); + } + TRY_STATUS(info1->load_special_creator_stat(val_pubkey, true)); + TRY_STATUS(info2->load_special_creator_stat(val_pubkey, true)); + td::TerminalIO::out() << "total: (" << info1->created_total.first << "," << info1->created_total.second << ") -> (" + << info2->created_total.first << "," << info2->created_total.second << ")\n"; + auto x = info2->created_total.first - info1->created_total.first; + auto y = info2->created_total.second - info1->created_total.second; + if (x <= 0 || y < 0 || (x | y) >= (1u << 31)) { + return td::Status::Error(PSLICE() << "impossible situation: zero or no blocks created: " << x + << " masterchain blocks, " << y << " shardchain blocks"); + } + auto x1 = info2->created_special.first - info1->created_special.first; + auto y1 = info2->created_special.second - info1->created_special.second; + if ((x1 | y1) < 0 || (x1 | y1) >= (1u << 31)) { + return td::Status::Error(PSLICE() << "impossible situation: validator " << val_pubkey.to_hex() << " created " << x1 + << " masterchain blocks, " << y1 << " shardchain blocks"); + } + td::TerminalIO::out() << "total: (" << x << "," << y << ")\n"; + try { + auto ccfg = block::Config::unpack_catchain_validators_config(info2->config->get_config_param(28)); + auto ccfg_old = block::Config::unpack_catchain_validators_config(info1->config->get_config_param(28)); + if (ccfg.shard_val_num != ccfg_old.shard_val_num || ccfg.shard_val_num <= 0) { + return td::Status::Error(PSLICE() << "shard validator group size changed from " << ccfg_old.shard_val_num + << " to " << ccfg.shard_val_num << ", or is not positive"); + } + int shard_count = ccfg.shard_val_num, main_count = info2->vset->main; + if (info1->vset->main != main_count || main_count <= 0) { + return td::Status::Error(PSLICE() << "masterchain validator group size changed from " << info1->vset->main + << " to " << main_count << ", or is not positive"); + } + double chunk_size = ccfg.shard_val_lifetime / 3. / shard_count; + block::MtCarloComputeShare shard_share(shard_count, info2->vset->export_scaled_validator_weights()); + + double xe = (val_idx < main_count ? (double)x / main_count : 0); + double ye = shard_share[val_idx] * (double)y / shard_count; + td::Bits256 pk = info2->vset->list[val_idx].pubkey.as_bits256(); + CHECK(pk == val_pubkey); + double p1 = create_prob((int)x1, .9 * xe), p2 = shard_create_prob((int)y1, .9 * ye, chunk_size); + td::TerminalIO::out() << "val #" << val_idx << ": pubkey " << pk.to_hex() << ", blocks created (" << x1 << "," << y1 + << "), expected (" << xe << "," << ye << "), probabilities " << p1 << " and " << p2 << "\n"; + bool severe = (rec.severity >= 2); + if (severe && std::min(p1, p2) < .00001) { + LOG(ERROR) << "validator #" << val_idx << " with pubkey " << pk.to_hex() + << " : serious misbehavior detected: created less than 90% of the expected amount of blocks with " + "probability 99.999% : created (" + << x1 << "," << y1 << "), expected (" << xe << "," << ye << ") masterchain/shardchain blocks\n"; + } else if (!severe && std::min(p1, p2) < .001) { + LOG(ERROR) << "validator #" << val_idx << " with pubkey " << pk.to_hex() + << " : moderate misbehavior detected: created less than 90% of the expected amount of blocks with " + "probability 99.9% : created (" + << x1 << "," << y1 << "), expected (" << xe << "," << ye << ") masterchain/shardchain blocks\n"; + } else { + LOG(ERROR) << "invalid (unsupported) complaint for validator #" << val_idx << " with pubkey " << pk.to_hex(); + show_vote(root->get_hash().bits(), false); + return td::Status::OK(); + } + auto suggested_fine = block::tlb::t_Grams.as_integer(rec.suggested_fine); + if (suggested_fine.is_null()) { + return td::Status::Error("cannot parse suggested fine"); + } + if (!check_punishment(interval, severe, suggested_fine, rec.suggested_fine_part)) { + LOG(ERROR) << "proposed punishment (fine " << td::dec_string(suggested_fine) + << ", fine_part=" << (double)rec.suggested_fine_part / (1LL << 32) << " is too harsh"; + show_vote(root->get_hash().bits(), false); + return td::Status::OK(); + } + LOG(INFO) << "accepting suggested punishment (affirmative vote)"; + show_vote(root->get_hash().bits(), true); + return td::Status::OK(); + } catch (vm::VmError& err) { + return err.as_status("vm error while scanning configuration proof:"); + } catch (vm::VmVirtError& err) { + return err.as_status("virtualization error while scanning configuration proof:"); + } +} + +td::Status TestNode::ValidatorLoadInfo::load_special_creator_stat(const td::Bits256& spec_pubkey, bool load_total) { + if (!vset) { + return td::Status::Error("no validator set loaded"); + } + int idx = vset->lookup_public_key(spec_pubkey); + if (idx < 0) { + return td::Status::Error("validator with public key "s + spec_pubkey.to_hex() + " not present in validator set"); + } + if (virt_root.is_null()) { + return td::Status::Error("no virtualized block state"); + } + try { + auto dict = block::get_block_create_stats_dict(virt_root); + if (!dict) { + return td::Status::Error("cannot extract BlockCreateStats from virtualized mc state"); + } + for (int i = 0; i <= (int)load_total; i++) { + td::Bits256 key = (i ? td::Bits256::zero() : spec_pubkey); + auto& p = *(i ? &created_total : &created_special); + auto cell = dict->lookup(key); + if (cell.is_null()) { + p.first = p.second = 0; + if (i) { + return td::Status::Error("no total created block statistics in BlockCreateStats"); + } + } else { + block::DiscountedCounter mc_cnt, shard_cnt; + if (!block::unpack_CreatorStats(std::move(cell), mc_cnt, shard_cnt)) { + return td::Status::Error(PSLICE() << "invalid CreatorStats record with key " << key.to_hex()); + } + p.first = mc_cnt.total; + p.second = shard_cnt.total; + } + } + special_idx = idx; + return td::Status::OK(); + } catch (vm::VmError& err) { + return err.as_status("vm error while extracting block creator data: "); + } catch (vm::VmVirtError& err) { + return err.as_status("virtualization error while extracting block creator data: "); + } +} + +td::Result> TestNode::ValidatorLoadInfo::preinit_from_producer_info( + Ref prod_info) { + if (prod_info.is_null()) { + return td::Status::Error("ProducerInfo cell is null"); + } + if (!block::gen::t_ProducerInfo.validate_ref(prod_info)) { + return td::Status::Error("invalid ProducerInfo"); + } + block::gen::ProducerInfo::Record rec; + ton::BlockIdExt blk_id; + ton::LogicalTime end_lt; + if (!(tlb::unpack_cell(prod_info, rec) && + block::tlb::t_ExtBlkRef.unpack(std::move(rec.mc_blk_ref), blk_id, &end_lt))) { + return td::Status::Error("cannot unpack ProducerInfo"); + } + auto info = std::make_unique(blk_id, std::move(rec.state_proof), std::move(rec.prod_proof)); + CHECK(info); + info->end_lt = end_lt; + info->block_created_at = rec.utime; + TRY_STATUS_PREFIX(info->init_check_proofs(), "error checking block/state proofs:"); + return std::move(info); +} + +td::Status TestNode::ValidatorLoadInfo::init_check_proofs() { + try { + ton::UnixTime utime; + ton::LogicalTime lt; + TRY_STATUS(check_header_proof(&utime, <)); + if (utime != block_created_at) { + return td::Status::Error(PSLICE() << "incorrect block creation time: declared " << block_created_at << ", actual " + << utime); + } + if (lt != end_lt) { + return td::Status::Error(PSLICE() << "incorrect block logical time: declared " << end_lt << ", actual " << lt); + } + auto vstate = vm::MerkleProof::virtualize(data_proof, 1); + if (vstate.is_null()) { + return td::Status::Error(PSLICE() << "cannot virtualize state of block " << blk_id.to_str()); + } + TRY_RESULT_PREFIX_ASSIGN(config, block::Config::extract_from_state(vstate, 0), "cannot unpack configuration:"); + auto vset_root = config->get_config_param(34); + if (vset_root.is_null()) { + vset_hash.set_zero(); + return td::Status::Error(PSLICE() << "no configuration parameter 34 (validator set) for block " + << blk_id.to_str()); + } + vset_hash = vset_root->get_hash().bits(); + virt_root = vstate; + return td::Status::OK(); + } catch (vm::VmError& err) { + return err.as_status("vm error:"); + } catch (vm::VmVirtError& err) { + return err.as_status("virtualization error:"); } } diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 24af7f81..6ebf0356 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -33,6 +33,7 @@ #include "vm/cells.h" #include "vm/stack.hpp" #include "block/block.h" +#include "block/mc-config.h" #include "td/utils/filesystem.h" using td::Ref; @@ -67,8 +68,8 @@ class TestNode : public td::actor::Actor { ton::BlockIdExt last_block_id_, last_state_id_; td::BufferSlice last_block_data_, last_state_data_; - ton::StdSmcAddress dns_root_; - bool dns_root_queried_{false}; + ton::StdSmcAddress dns_root_, elect_addr_; + bool dns_root_queried_{false}, elect_addr_queried_{false}; std::string line_; const char *parse_ptr_, *parse_end_; @@ -89,6 +90,9 @@ class TestNode : public td::actor::Actor { std::unique_ptr make_callback(); + using creator_stats_func_t = + std::function; + struct TransId { ton::Bits256 acc_addr; ton::LogicalTime trans_lt; @@ -98,6 +102,75 @@ class TestNode : public td::actor::Actor { } }; + struct BlockHdrInfo { + ton::BlockIdExt blk_id; + Ref proof, virt_blk_root; + int mode; + BlockHdrInfo() : mode(-1) { + } + BlockHdrInfo(const ton::BlockIdExt blk_id_, Ref proof_, Ref vroot_, int mode_) + : blk_id(blk_id_), proof(std::move(proof_)), virt_blk_root(std::move(vroot_)), mode(mode_) { + } + }; + + struct ConfigInfo { + std::unique_ptr config; + Ref state_proof, config_proof; + ConfigInfo() = default; + ConfigInfo(std::unique_ptr config_, Ref state_proof_, Ref config_proof_) + : config(std::move(config_)), state_proof(std::move(state_proof_)), config_proof(std::move(config_proof_)) { + } + }; + + struct CreatorStatsRes { + int mode; + bool complete{false}; + td::Bits256 last_key; + Ref state_proof, data_proof; + CreatorStatsRes(int mode_ = 0) : mode(mode_) { + last_key.set_zero(); + } + CreatorStatsRes(int mode_, const td::Bits256& key_, Ref st_proof_ = {}, Ref dproof_ = {}) + : mode(mode_), last_key(key_), state_proof(std::move(st_proof_)), data_proof(std::move(dproof_)) { + } + }; + + struct ValidatorLoadInfo { + ton::BlockIdExt blk_id; + Ref state_proof, data_proof, virt_root; + std::unique_ptr config; + ton::UnixTime block_created_at{0}; + ton::UnixTime valid_since{0}; + ton::LogicalTime end_lt{0}; + ton::Bits256 vset_hash; + Ref vset_root; + std::unique_ptr vset; + std::map vset_map; + int special_idx{-1}; + std::pair created_total, created_special; + std::vector> created; + ValidatorLoadInfo(ton::BlockIdExt blkid, Ref root, Ref root2, + std::unique_ptr cfg = {}) + : blk_id(blkid) + , state_proof(std::move(root)) + , data_proof(std::move(root2)) + , config(std::move(cfg)) + , valid_since(0) { + } + td::Status unpack_vset(); + bool store_record(const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt); + bool has_data() const { + return blk_id.is_masterchain_ext() && state_proof.not_null() && data_proof.not_null() && config; + } + td::Status check_header_proof(ton::UnixTime* save_utime = nullptr, ton::LogicalTime* save_lt = nullptr) const; + td::Result> build_proof(int idx, td::Bits256* save_pubkey = nullptr) const; + td::Result> build_producer_info(int idx, td::Bits256* save_pubkey = nullptr) const; + td::Status init_check_proofs(); + static td::Result> preinit_from_producer_info(Ref prod_info); + td::Status load_special_creator_stat(const td::Bits256& spec_pubkey, bool load_total = true); + }; + void run_init_queries(); char cur() const { return *parse_ptr_; @@ -133,6 +206,8 @@ class TestNode : public td::actor::Actor { std::vector params, td::BufferSlice remote_c7, td::BufferSlice remote_libs, td::BufferSlice remote_result, int remote_exit_code, td::Promise> promise); + bool register_config_param(int idx, Ref value); + bool register_config_param1(Ref value); bool register_config_param4(Ref value); bool dns_resolve_start(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt blkid, std::string domain, int cat, int mode); @@ -144,18 +219,24 @@ class TestNode : public td::actor::Actor { bool show_dns_record(std::ostream& os, int cat, Ref value, bool raw_dump); bool get_all_shards(std::string filename = "", bool use_last = true, ton::BlockIdExt blkid = {}); void got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::BufferSlice data, std::string filename); - bool get_config_params(ton::BlockIdExt blkid, td::Promise do_after, int mode = 0, std::string filename = "", - std::vector params = {}); - void got_config_params(ton::BlockIdExt req_blkid, ton::BlockIdExt blkid, td::BufferSlice state_proof, - td::BufferSlice cfg_proof, int mode, std::string filename, std::vector params, - td::Promise do_after); + bool parse_get_config_params(ton::BlockIdExt blkid, int mode = 0, std::string filename = "", + std::vector params = {}); + bool get_config_params(ton::BlockIdExt blkid, td::Promise> promise, int mode = 0, + std::string filename = "", std::vector params = {}); + bool get_config_params_ext(ton::BlockIdExt blkid, td::Promise promise, int mode = 0, + std::string filename = "", std::vector params = {}); + void got_config_params(ton::BlockIdExt req_blkid, int mode, std::string filename, std::vector params, + td::Result R, td::Promise promise); bool get_block(ton::BlockIdExt blk, bool dump = false); void got_block(ton::BlockIdExt blkid, td::BufferSlice data, bool dump); bool get_state(ton::BlockIdExt blk, bool dump = false); void got_state(ton::BlockIdExt blkid, ton::RootHash root_hash, ton::FileHash file_hash, td::BufferSlice data, bool dump); - bool get_block_header(ton::BlockIdExt blk, int mode); - bool lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg); + bool get_show_block_header(ton::BlockIdExt blk, int mode); + bool get_block_header(ton::BlockIdExt blk, int mode, td::Promise promise); + bool lookup_show_block(ton::ShardIdFull shard, int mode, td::uint64 arg); + bool lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg, td::Promise); + void got_block_header_raw(td::BufferSlice res, td::Promise promise, ton::BlockIdExt req_blkid = {}); void got_block_header(ton::BlockIdExt blkid, td::BufferSlice data, int mode); bool show_block_header(ton::BlockIdExt blkid, Ref root, int mode); bool show_state_header(ton::BlockIdExt blkid, Ref root, int mode); @@ -177,9 +258,43 @@ class TestNode : public td::actor::Actor { void got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice res); bool get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_count, ton::Bits256 start_after, ton::UnixTime min_utime); - void got_creator_stats(ton::BlockIdExt req_blkid, ton::BlockIdExt blkid, int req_mode, int mode, - td::Bits256 start_after, ton::UnixTime min_utime, td::BufferSlice state_proof, - td::BufferSlice data_proof, int count, int req_count, bool complete); + bool get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_count, ton::Bits256 start_after, + ton::UnixTime min_utime, creator_stats_func_t func, td::Promise promise); + bool get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton::UnixTime min_utime, creator_stats_func_t func, + std::unique_ptr state, td::Promise> promise); + void got_creator_stats(ton::BlockIdExt req_blkid, ton::BlockIdExt blkid, int mode, ton::UnixTime min_utime, + td::BufferSlice state_proof, td::BufferSlice data_proof, int count, int req_count, + bool complete, creator_stats_func_t func, std::unique_ptr state, + td::Promise> promise); + bool check_validator_load(int start_time, int end_time, int mode = 0, std::string file_pfx = ""); + void continue_check_validator_load(ton::BlockIdExt blkid1, Ref root1, ton::BlockIdExt blkid2, + Ref root2, int mode = 0, std::string file_pfx = ""); + void continue_check_validator_load2(std::unique_ptr info1, + std::unique_ptr info2, int mode = 0, + std::string file_pfx = ""); + void continue_check_validator_load3(std::unique_ptr info1, + std::unique_ptr info2, int mode = 0, + std::string file_pfx = ""); + td::Status write_val_create_proof(ValidatorLoadInfo& info1, ValidatorLoadInfo& info2, int idx, bool severe, + std::string file_pfx, int cnt); + bool load_creator_stats(std::unique_ptr load_to, + td::Promise> promise, bool need_proofs); + td::Status check_validator_load_proof(std::string filename, std::string vset_filename = "", + ton::Bits256 vset_hash = ton::Bits256::zero()); + td::Status continue_check_validator_load_proof(std::unique_ptr info1, + std::unique_ptr info2, Ref root); + bool get_elector_addr(td::Promise promise); + bool get_past_validator_sets(); + bool send_past_vset_query(ton::StdSmcAddress elector_addr); + void register_past_vset_info(vm::StackEntry list); + bool get_complaints(unsigned elect_id, std::string file_pfx); + void send_get_complaints_query(unsigned elect_id, ton::StdSmcAddress elector_addr, std::string file_pfx); + void save_complaints(unsigned elect_id, Ref complaints, std::string file_pfx); + td::Status get_complaint_price(unsigned expires_in, std::string filename); + td::Status get_complaint_price(unsigned expires_in, unsigned bits, unsigned refs, + td::Bits256 chash = td::Bits256::zero(), std::string filename = ""); + void send_compute_complaint_price_query(ton::StdSmcAddress elector_addr, unsigned expires_in, unsigned bits, + unsigned refs, td::Bits256 chash, std::string filename); bool cache_cell(Ref cell); bool list_cached_cells() const; bool dump_cached_cell(td::Slice hash_pfx, td::Slice type_name = {}); @@ -222,6 +337,17 @@ class TestNode : public td::actor::Actor { bool show_new_blkids(bool all = false); bool complete_blkid(ton::BlockId partial_blkid, ton::BlockIdExt& complete_blkid) const; td::Promise trivial_promise(); + template + td::Promise trivial_promise_of() { + return td::PromiseCreator::lambda([Self = actor_id(this)](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << "error: " << res.move_as_error(); + } + }); + } + static ton::UnixTime now() { + return static_cast(td::Clocks::system()); + } static const tlb::TypenameLookup& get_tlb_dict(); public: @@ -292,7 +418,8 @@ class TestNode : public td::actor::Actor { //td::actor::SchedulerContext::get()->stop(); } - void got_result(); + void got_result(td::Result R, td::Promise promise); + void after_got_result(bool ok); bool envelope_send_query(td::BufferSlice query, td::Promise promise); void parse_line(td::BufferSlice data); diff --git a/tdactor/td/actor/PromiseFuture.h b/tdactor/td/actor/PromiseFuture.h index 022a98e3..2b8890d3 100644 --- a/tdactor/td/actor/PromiseFuture.h +++ b/tdactor/td/actor/PromiseFuture.h @@ -44,8 +44,24 @@ class GetArg { using type = Arg; }; +template +struct GetRet : public GetRet {}; + +template +class GetRet { + public: + using type = R; +}; +template +class GetRet { + public: + using type = R; +}; + template using get_arg_t = std::decay_t::type>; +template +using get_ret_t = std::decay_t::type>; template struct DropResult { @@ -131,6 +147,7 @@ constexpr bool is_promise_interface_ptr() { template class LambdaPromise : public PromiseInterface { public: + using ArgT = ValueT; void set_value(ValueT &&value) override { CHECK(has_lambda_.get()); do_ok(std::move(value)); @@ -288,12 +305,6 @@ class Promise { std::unique_ptr> promise_; }; -template -auto make_promise(F &&f) { - using ValueT = detail::drop_result_t>; - return Promise(promise_interface_ptr(std::forward(f))); -} - namespace detail { template class JoinPromise : public PromiseInterface { @@ -331,6 +342,16 @@ class PromiseCreator { } }; +template +auto make_promise(F &&f) { + using ValueT = typename decltype(PromiseCreator::lambda(std::move(f)))::ArgT; + return Promise(PromiseCreator::lambda(std::move(f))); +} +template +auto make_promise(Promise &&f) { + return std::move(f); +} + template class SafePromise { public: @@ -356,4 +377,145 @@ class SafePromise { Promise promise_; Result result_; }; + +template +class PromiseMerger; + +template +struct SplitPromise { + using PromiseT = decltype(make_promise(std::declval())); + using ArgT = typename PromiseT::ArgT; + + template + static std::pair, Promise> split(std::pair); + template + static std::tuple...> split(std::tuple); + using SplittedT = decltype(split(std::declval())); + + template + static PromiseMerger merger(std::pair); + template + static PromiseMerger merger(std::tuple); + using MergerT = decltype(merger(std::declval())); +}; + +template +class PromiseMerger : public std::enable_shared_from_this> { + public: + std::tuple...> args_; + PromiseT promise_; + + PromiseMerger(PromiseT promise) : promise_(std::move(promise)) { + } + ~PromiseMerger() { + td::Status status; + tuple_for_each(args_, [&status](auto &&arg) { + if (status.is_error()) { + return; + } + if (arg.is_error()) { + status = arg.move_as_error(); + } + }); + if (status.is_error()) { + promise_.set_error(std::move(status)); + return; + } + call_tuple([this](auto &&... args) { promise_.set_value({args.move_as_ok()...}); }, std::move(args_)); + } + + template + Promise make_promise(T &arg) { + return [&arg, self = this->shared_from_this()](auto res) { arg = std::move(res); }; + } + + template + auto split() { + return call_tuple([this](auto &&... arg) { return R{this->make_promise(arg)...}; }, std::move(args_)); + } +}; + +template +auto split_promise(F &&f) { + auto merger = std::make_shared::MergerT>(std::move(f)); + return merger->template split::SplittedT>(); +} + +template +struct PromiseFuture { + Result> promise_; + Result result_; + ~PromiseFuture() { + if (promise_.is_ok()) { + promise_.move_as_ok().set_result(std::move(result_)); + } else { + LOG(ERROR) << "Lost PromiseFuture"; + } + } +}; +template +struct Future; + +template +std::pair, Future> make_promise_future(); + +template +struct Future { + Promise> promise_; + Future(Promise> promise) : promise_(std::move(promise)) { + } + + void finish(Promise promise) { + promise_.set_value(std::move(promise)); + } + + template + auto map(F &&f) { + using R = detail::drop_result_t()))>; + auto pf = make_promise_future(); + promise_.set_value([p = std::move(pf.first), f = std::move(f)](Result res) mutable { + TRY_RESULT_PROMISE(p, x, std::move(res)); + p.set_result(f(std::move(x))); + }); + + return std::move(pf.second); + } + + template + auto fmap(F &&f) { + return flatten(map(std::move(f))); + } + + template + static Future flatten(Future> ff) { + auto pf = make_promise_future(); + ff.promise_.set_value([p = std::move(pf.first)](Result> r_f) mutable { + TRY_RESULT_PROMISE(p, f, std::move(r_f)); + // Promise p + // Future f + f.promise_.set_value(std::move(p)); + }); + return std::move(pf.second); + } +}; + +template +Future make_future(T &&value) { + return Future([value = std::move(value)](Result> r_promise) mutable { + if (r_promise.is_ok()) { + r_promise.move_as_ok().set_value(std::move(value)); + } else { + LOG(ERROR) << "Lost future"; + } + }); +} + +template +std::pair, Future> make_promise_future() { + auto pf = std::make_shared>(); + Future future([pf](Result> res) mutable { pf->promise_ = std::move(res); }); + Promise promise = [pf = std::move(pf)](Result res) mutable { pf->result_ = std::move(res); }; + return std::make_pair(std::move(promise), std::move(future)); +} + } // namespace td diff --git a/tdactor/td/actor/actor.h b/tdactor/td/actor/actor.h index 5f699588..1f4f6e99 100644 --- a/tdactor/td/actor/actor.h +++ b/tdactor/td/actor/actor.h @@ -100,6 +100,28 @@ void send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { #endif +template , + size_t argument_count = member_function_argument_count(), + std::enable_if_t with_promise = false> +auto future_send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { + using R = ::td::detail::get_ret_t>; + auto pf = make_promise_future(); + send_closure(std::forward(actor_id), std::move(function), std::forward(args)..., + std::move(pf.first)); + return std::move(pf.second); +} + +template , + size_t argument_count = member_function_argument_count(), + std::enable_if_t with_promise = true> +Future future_send_closure(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { + auto pf = make_promise_future(); + send_closure(std::forward(actor_id), std::move(function), std::forward(args)..., + std::move(pf.first)); + return std::move(pf.second); +} + template bool send_closure_bool(ActorIdT &&actor_id, FunctionT function, ArgsT &&... args) { send_closure(std::forward(actor_id), function, std::forward(args)...); diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index 33e259a2..eacc6192 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -63,6 +63,36 @@ class ActorSignals { using core::Actor; using core::SchedulerContext; using core::SchedulerId; +using core::set_debug; + +struct Debug { + public: + Debug() = default; + Debug(std::shared_ptr group_info) : group_info_(std::move(group_info)) { + } + template + void for_each(F &&f) { + for (auto &scheduler : group_info_->schedulers) { + f(scheduler.io_worker->debug); + for (auto &cpu : scheduler.cpu_workers) { + f(cpu->debug); + } + } + } + + void dump() { + for_each([](core::Debug &debug) { + core::DebugInfo info; + debug.read(info); + if (info.is_active) { + LOG(ERROR) << info.name << " " << td::format::as_time(Time::now() - info.start_at); + } + }); + } + + private: + std::shared_ptr group_info_; +}; class Scheduler { public: @@ -110,6 +140,10 @@ class Scheduler { } } + Debug get_debug() { + return Debug{group_info_}; + } + bool run() { start(); while (schedulers_[0]->run(10)) { diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index d752560d..d78660c6 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -32,6 +32,7 @@ void CpuWorker::run() { MpmcWaiter::Slot slot; waiter_.init_slot(slot, thread_id); + auto &debug = dispatcher.get_debug(); while (true) { SchedulerMessage message; if (try_pop(message, thread_id)) { @@ -39,6 +40,7 @@ void CpuWorker::run() { if (!message) { return; } + auto lock = debug.start(message->get_name()); ActorExecutor executor(*message, dispatcher, ActorExecutor::Options().with_from_queue()); } else { waiter_.wait(slot); diff --git a/tdactor/td/actor/core/IoWorker.cpp b/tdactor/td/actor/core/IoWorker.cpp index d1f10c63..ee3576bd 100644 --- a/tdactor/td/actor/core/IoWorker.cpp +++ b/tdactor/td/actor/core/IoWorker.cpp @@ -19,6 +19,7 @@ #include "td/actor/core/IoWorker.h" #include "td/actor/core/ActorExecutor.h" +#include "td/actor/core/Scheduler.h" namespace td { namespace actor { @@ -42,6 +43,7 @@ bool IoWorker::run_once(double timeout) { auto &poll = SchedulerContext::get()->get_poll(); #endif auto &heap = SchedulerContext::get()->get_heap(); + auto &debug = SchedulerContext::get()->get_debug(); auto now = Time::now(); // update Time::now_cached() while (!heap.empty() && heap.top_key() <= now) { @@ -49,6 +51,7 @@ bool IoWorker::run_once(double timeout) { auto *actor_info = ActorInfo::from_heap_node(heap_node); auto id = actor_info->unpin(); + auto lock = debug.start(actor_info->get_name()); ActorExecutor executor(*actor_info, dispatcher, ActorExecutor::Options().with_has_poll(true)); if (executor.can_send_immediate()) { executor.send_immediate(ActorSignals::one(ActorSignals::Alarm)); @@ -68,6 +71,7 @@ bool IoWorker::run_once(double timeout) { dispatcher.set_alarm_timestamp(message); continue; } + auto lock = debug.start(message->get_name()); ActorExecutor executor(*message, dispatcher, ActorExecutor::Options().with_from_queue().with_has_poll(true)); } queue_.reader_flush(); diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index 0863bda0..f79620d9 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -25,6 +25,15 @@ namespace td { namespace actor { namespace core { +std::atomic debug; +void set_debug(bool flag) { + debug = flag; +} + +bool need_debug() { + return debug.load(std::memory_order_relaxed); +} + Scheduler::Scheduler(std::shared_ptr scheduler_group_info, SchedulerId id, size_t cpu_threads_count) : scheduler_group_info_(std::move(scheduler_group_info)), cpu_threads_(cpu_threads_count) { scheduler_group_info_->active_scheduler_count++; @@ -128,13 +137,14 @@ void Scheduler::do_stop() { } Scheduler::ContextImpl::ContextImpl(ActorInfoCreator *creator, SchedulerId scheduler_id, CpuWorkerId cpu_worker_id, - SchedulerGroupInfo *scheduler_group, Poll *poll, KHeap *heap) + SchedulerGroupInfo *scheduler_group, Poll *poll, KHeap *heap, Debug *debug) : creator_(creator) , scheduler_id_(scheduler_id) , cpu_worker_id_(cpu_worker_id) , scheduler_group_(scheduler_group) , poll_(poll) - , heap_(heap) { + , heap_(heap) + , debug_(debug) { } SchedulerId Scheduler::ContextImpl::get_scheduler_id() const { @@ -184,6 +194,9 @@ KHeap &Scheduler::ContextImpl::get_heap() { CHECK(has_heap()); return *heap_; } +Debug &Scheduler::ContextImpl::get_debug() { + return *debug_; +} void Scheduler::ContextImpl::set_alarm_timestamp(const ActorInfoPtr &actor_info_ptr) { // Ideas for optimization diff --git a/tdactor/td/actor/core/Scheduler.h b/tdactor/td/actor/core/Scheduler.h index 3cc627ee..377d835f 100644 --- a/tdactor/td/actor/core/Scheduler.h +++ b/tdactor/td/actor/core/Scheduler.h @@ -31,6 +31,7 @@ #include "td/actor/core/SchedulerId.h" #include "td/actor/core/SchedulerMessage.h" +#include "td/utils/AtomicRead.h" #include "td/utils/Closure.h" #include "td/utils/common.h" #include "td/utils/format.h" @@ -65,6 +66,54 @@ namespace actor { namespace core { class IoWorker; +struct DebugInfo { + bool is_active{false}; + double start_at{0}; + static constexpr size_t name_size{32}; + char name[name_size] = {}; + void set_name(td::Slice from) { + from.truncate(name_size - 1); + std::memcpy(name, from.data(), from.size()); + name[from.size()] = 0; + } +}; + +void set_debug(bool flag); +bool need_debug(); + +struct Debug { + public: + bool is_on() const { + return need_debug(); + } + struct Destructor { + void operator()(Debug *info) { + info->info_.lock().value().is_active = false; + } + }; + + void read(DebugInfo &info) { + info_.read(info); + } + + std::unique_ptr start(td::Slice name) { + if (!is_on()) { + return {}; + } + { + auto lock = info_.lock(); + auto &value = lock.value(); + value.is_active = true; + value.start_at = Time::now(); + value.set_name(name); + } + return std::unique_ptr(this); + } + + private: + AtomicRead info_; +}; + struct WorkerInfo { enum class Type { Io, Cpu } type{Type::Io}; WorkerInfo() = default; @@ -73,6 +122,7 @@ struct WorkerInfo { } ActorInfoCreator actor_info_creator; CpuWorkerId cpu_worker_id; + Debug debug; }; template @@ -195,7 +245,7 @@ class Scheduler { class ContextImpl : public SchedulerContext { public: ContextImpl(ActorInfoCreator *creator, SchedulerId scheduler_id, CpuWorkerId cpu_worker_id, - SchedulerGroupInfo *scheduler_group, Poll *poll, KHeap *heap); + SchedulerGroupInfo *scheduler_group, Poll *poll, KHeap *heap, Debug *debug); SchedulerId get_scheduler_id() const override; void add_to_queue(ActorInfoPtr actor_info_ptr, SchedulerId scheduler_id, bool need_poll) override; @@ -208,6 +258,8 @@ class Scheduler { bool has_heap() override; KHeap &get_heap() override; + Debug &get_debug() override; + void set_alarm_timestamp(const ActorInfoPtr &actor_info_ptr) override; bool is_stop_requested() override; @@ -225,6 +277,8 @@ class Scheduler { Poll *poll_; KHeap *heap_; + + Debug *debug_; }; template @@ -234,7 +288,8 @@ class Scheduler { #endif bool is_io_worker = worker_info.type == WorkerInfo::Type::Io; ContextImpl context(&worker_info.actor_info_creator, info_->id, worker_info.cpu_worker_id, - scheduler_group_info_.get(), is_io_worker ? &poll_ : nullptr, is_io_worker ? &heap_ : nullptr); + scheduler_group_info_.get(), is_io_worker ? &poll_ : nullptr, is_io_worker ? &heap_ : nullptr, + &worker_info.debug); SchedulerContext::Guard guard(&context); f(); } diff --git a/tdactor/td/actor/core/SchedulerContext.h b/tdactor/td/actor/core/SchedulerContext.h index 49cbf63d..e46aef7d 100644 --- a/tdactor/td/actor/core/SchedulerContext.h +++ b/tdactor/td/actor/core/SchedulerContext.h @@ -37,6 +37,7 @@ class SchedulerDispatcher { virtual void set_alarm_timestamp(const ActorInfoPtr &actor_info_ptr) = 0; }; +struct Debug; class SchedulerContext : public Context, public SchedulerDispatcher { public: virtual ~SchedulerContext() = default; @@ -55,6 +56,9 @@ class SchedulerContext : public Context, public SchedulerDispa // Stop all schedulers virtual bool is_stop_requested() = 0; virtual void stop() = 0; + + // Debug + virtual Debug &get_debug() = 0; }; } // namespace core } // namespace actor diff --git a/tdactor/test/actors_core.cpp b/tdactor/test/actors_core.cpp index 1ec9dda2..ae10eb9b 100644 --- a/tdactor/test/actors_core.cpp +++ b/tdactor/test/actors_core.cpp @@ -675,7 +675,8 @@ TEST(Actor2, actor_function_result) { public: A(std::shared_ptr watcher) : watcher_(std::move(watcher)) { } - void on_result(uint32 x, uint32 y) { + void on_result(uint32 x, td::Result r_y) { + auto y = r_y.move_as_ok(); LOG_CHECK(x * x == y) << x << " " << y; if (--cnt_ == 0) { stop(); @@ -683,7 +684,7 @@ TEST(Actor2, actor_function_result) { } void start_up() { b_ = create_actor(ActorOptions().with_name("B")); - cnt_ = 3; + cnt_ = 5; send_closure(b_, &B::query, 3, [a = std::make_unique(), self = actor_id(this)](td::Result y) { LOG_IF(ERROR, y.is_error()) << y.error(); send_closure(self, &A::on_result, 3, y.ok()); @@ -696,6 +697,11 @@ TEST(Actor2, actor_function_result) { CHECK(!self.empty()); send_closure(self, &A::on_result, 5, y); }); + auto future = future_send_closure(b_, &B::query, 7); + future.finish(td::promise_send_closure(actor_id(this), &A::on_result, 7)); + //TODO: deduce Future type (i.e. Future) + auto future2 = future_send_closure(b_, &B::query_async, 7); + future2.finish(td::promise_send_closure(actor_id(this), &A::on_result, 7)); } private: @@ -714,12 +720,12 @@ TEST(Actor2, actor_function_result) { } TEST(Actor2, actor_ping_pong) { - auto group_info = std::make_shared(1); - core::Scheduler scheduler{group_info, SchedulerId{0}, 3}; + Scheduler scheduler{{3}, Scheduler::Paused}; sb.clear(); scheduler.start(); auto watcher = td::create_shared_destructor([] { SchedulerContext::get()->stop(); }); + td::actor::set_debug(true); for (int i = 0; i < 2000; i++) { scheduler.run_in_context([watcher] { class PingPong : public Actor { @@ -781,9 +787,9 @@ TEST(Actor2, actor_ping_pong) { }); } watcher.reset(); - while (scheduler.run(1000)) { + while (scheduler.run(0.1)) { + //scheduler.get_debug().dump(); } - core::Scheduler::close_scheduler_group(*group_info); sb.clear(); } diff --git a/tdactor/test/actors_promise.cpp b/tdactor/test/actors_promise.cpp index 5921f725..f1d57069 100644 --- a/tdactor/test/actors_promise.cpp +++ b/tdactor/test/actors_promise.cpp @@ -135,6 +135,78 @@ TEST(Actor, safe_promise) { ASSERT_EQ(res, 3); } +TEST(Actor, split_promise) { + using td::Promise; + using td::Result; + using td::split_promise; + using td::SplitPromise; + { + td::optional> x; + auto pair = [&](Result> res) { x = res.move_as_ok(); }; + static_assert(std::is_same::ArgT, std::pair>::value, "A"); + static_assert( + std::is_same::SplittedT, std::pair, Promise>>::value, "A"); + auto splitted = split_promise(pair); + static_assert(std::is_same, Promise>>::value, "A"); + + splitted.first.set_value(1); + splitted.second.set_value(2.0); + CHECK(x.unwrap() == std::make_pair(1, 2.0)); + } // namespace td + { + td::optional> x; + auto triple = [&](Result> res) { x = res.move_as_ok(); }; + static_assert(std::is_same::ArgT, std::tuple>::value, "A"); + static_assert(std::is_same::SplittedT, + std::tuple, Promise, Promise>>::value, + "A"); + auto splitted = split_promise(triple); + static_assert( + std::is_same, Promise, Promise>>::value, "A"); + std::get<0>(splitted).set_value(1); + std::get<1>(splitted).set_value(2.0); + std::get<2>(splitted).set_value("hello"); + CHECK(x.unwrap() == std::make_tuple(1, 2.0, "hello")); + } + { + int code = 0; + auto pair = [&](Result> res) { + res.ensure_error(); + code = res.error().code(); + }; + auto splitted = split_promise(td::Promise>(pair)); + splitted.second.set_error(td::Status::Error(123, "123")); + CHECK(code == 0); + splitted.first.set_value(1); + CHECK(code == 123); + } +} + +TEST(Actor, promise_future) { + using td::make_promise_future; + { + auto pf = make_promise_future(); + td::optional res; + pf.second.map([](int x) { return x * 2; }).map([](int x) { return x + 10; }).map([&](int x) { + res = x; + return td::Unit(); + }); + CHECK(!res); + pf.first.set_value(6); + ASSERT_EQ(22, res.unwrap()); + } + { + LOG(ERROR) << "Second test"; + td::optional res; + td::make_future(6) + .map([](int x) { return x * 2; }) + .map([](int x) { return x + 10; }) + .fmap([&](int x) { return td::make_future(x * 2); }) + .finish([&](int x) { res = x; }); + ASSERT_EQ(44, res.unwrap()); + } +} + TEST(Actor2, actor_lost_promise) { using namespace td::actor; using namespace td; @@ -459,7 +531,7 @@ class SampleActor : public Actor { detail::current_actor().print_a(); co_await OnActor(self); LOG(ERROR) << "exit print_a"; - co_return{}; + co_return {}; } task print_b() { auto self = actor_id(this); @@ -468,7 +540,7 @@ class SampleActor : public Actor { detail::current_actor().print_b(); co_await OnActor(self); LOG(ERROR) << "exit print_b"; - co_return{}; + co_return {}; } immediate_task run_coroutine() { diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index 088958f6..4e0d8538 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -55,6 +55,10 @@ class KeyValue : public KeyValueReader { virtual Status set(Slice key, Slice value) = 0; virtual Status erase(Slice key) = 0; + virtual Status begin_write_batch() = 0; + virtual Status commit_write_batch() = 0; + virtual Status abort_write_batch() = 0; + virtual Status begin_transaction() = 0; virtual Status commit_transaction() = 0; virtual Status abort_transaction() = 0; @@ -86,6 +90,16 @@ class PrefixedKeyValue : public KeyValue { return kv_->erase(PSLICE() << prefix_ << key); } + Status begin_write_batch() override { + return kv_->begin_write_batch(); + } + Status commit_write_batch() override { + return kv_->commit_write_batch(); + } + Status abort_write_batch() override { + return kv_->abort_write_batch(); + } + Status begin_transaction() override { return kv_->begin_transaction(); } diff --git a/tddb/td/db/MemoryKeyValue.cpp b/tddb/td/db/MemoryKeyValue.cpp index 473ee780..aaf5472d 100644 --- a/tddb/td/db/MemoryKeyValue.cpp +++ b/tddb/td/db/MemoryKeyValue.cpp @@ -61,6 +61,15 @@ std::unique_ptr MemoryKeyValue::snapshot() { std::string MemoryKeyValue::stats() const { return PSTRING() << "MemoryKeyValueStats{" << tag("get_count", get_count_) << "}"; } +Status MemoryKeyValue::begin_write_batch() { + UNREACHABLE(); +} +Status MemoryKeyValue::commit_write_batch() { + UNREACHABLE(); +} +Status MemoryKeyValue::abort_write_batch() { + UNREACHABLE(); +} Status MemoryKeyValue::begin_transaction() { UNREACHABLE(); diff --git a/tddb/td/db/MemoryKeyValue.h b/tddb/td/db/MemoryKeyValue.h index ee6fd36d..c9d584bc 100644 --- a/tddb/td/db/MemoryKeyValue.h +++ b/tddb/td/db/MemoryKeyValue.h @@ -29,6 +29,10 @@ class MemoryKeyValue : public KeyValue { Status erase(Slice key) override; Result count(Slice prefix) override; + Status begin_write_batch() override; + Status commit_write_batch() override; + Status abort_write_batch() override; + Status begin_transaction() override; Status commit_transaction() override; Status abort_transaction() override; diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index e99cd436..500985e2 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -78,7 +78,18 @@ Result RocksDb::open(std::string path) { options.bytes_per_sync = 1 << 20; options.writable_file_max_buffer_size = 2 << 14; options.statistics = statistics; - TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(options, std::move(path), &db))); + rocksdb::OptimisticTransactionDBOptions occ_options; + occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; + rocksdb::ColumnFamilyOptions cf_options(options); + std::vector column_families; + column_families.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, cf_options)); + std::vector handles; + TRY_STATUS(from_rocksdb( + rocksdb::OptimisticTransactionDB::Open(options, occ_options, std::move(path), column_families, &handles, &db))); + CHECK(handles.size() == 1); + // i can delete the handle since DBImpl is always holding a reference to + // default column family + delete handles[0]; } return RocksDb(std::shared_ptr(db), std::move(statistics)); } @@ -161,31 +172,41 @@ Result RocksDb::count(Slice prefix) { return res; } +Status RocksDb::begin_write_batch() { + CHECK(!transaction_); + write_batch_ = std::make_unique(); + return Status::OK(); +} + Status RocksDb::begin_transaction() { - //write_batch_ = std::make_unique(); + CHECK(!write_batch_); rocksdb::WriteOptions options; options.sync = true; transaction_.reset(db_->BeginTransaction(options, {})); return Status::OK(); } -Status RocksDb::commit_transaction() { - //CHECK(write_batch_); - //auto write_batch = std::move(write_batch_); - //rocksdb::WriteOptions options; - //options.sync = true; - //TRY_STATUS(from_rocksdb(db_->Write(options, write_batch.get()))); - //return Status::OK(); +Status RocksDb::commit_write_batch() { + CHECK(write_batch_); + auto write_batch = std::move(write_batch_); + rocksdb::WriteOptions options; + options.sync = true; + return from_rocksdb(db_->Write(options, write_batch.get())); +} +Status RocksDb::commit_transaction() { CHECK(transaction_); - auto res = from_rocksdb(transaction_->Commit()); - transaction_.reset(); - return res; + auto transaction = std::move(transaction_); + return from_rocksdb(transaction->Commit()); +} + +Status RocksDb::abort_write_batch() { + CHECK(write_batch_); + write_batch_.reset(); + return Status::OK(); } Status RocksDb::abort_transaction() { - //CHECK(write_batch_); - //write_batch_.reset(); CHECK(transaction_); transaction_.reset(); return Status::OK(); diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 3745701b..b8bfaf9d 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -45,6 +45,10 @@ class RocksDb : public KeyValue { Status erase(Slice key) override; Result count(Slice prefix) override; + Status begin_write_batch() override; + Status commit_write_batch() override; + Status abort_write_batch() override; + Status begin_transaction() override; Status commit_transaction() override; Status abort_transaction() override; diff --git a/tdutils/td/utils/AtomicRead.h b/tdutils/td/utils/AtomicRead.h new file mode 100644 index 00000000..0eb85c6d --- /dev/null +++ b/tdutils/td/utils/AtomicRead.h @@ -0,0 +1,90 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2019-2020 Telegram Systems LLP +*/ +#include + +#include "td/utils/common.h" +#include "td/utils/port/thread.h" +namespace td { +template + +class AtomicRead { + public: + void read(T &dest) const { + while (true) { + static_assert(std::is_trivially_copyable::value, "T must be trivially copyable"); + auto version_before = version.load(); + memcpy(&dest, &value, sizeof(dest)); + auto version_after = version.load(); + if (version_before == version_after && version_before % 2 == 0) { + break; + } + td::this_thread::yield(); + } + } + + friend struct Write; + struct Write { + explicit Write(AtomicRead *read) { + read->do_lock(); + ptr.reset(read); + } + struct Destructor { + void operator()(AtomicRead *read) const { + read->do_unlock(); + } + }; + T &operator*() { + return value(); + } + T *operator->() { + return &value(); + } + T &value() { + CHECK(ptr); + return ptr->value; + } + + private: + std::unique_ptr ptr; + }; + Write lock() { + return Write(this); + } + + private: + std::atomic version{0}; + T value; + + void do_lock() { + CHECK(++version % 2 == 1); + } + void do_unlock() { + CHECK(++version % 2 == 0); + } +}; +}; // namespace td diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index 8016099f..a9879996 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -57,6 +57,15 @@ } \ } +#define TRY_STATUS_PROMISE_PREFIX(promise_name, status, prefix) \ + { \ + auto try_status = (status); \ + if (try_status.is_error()) { \ + promise_name.set_error(try_status.move_as_error_prefix(prefix)); \ + return; \ + } \ + } + #define TRY_RESULT(name, result) TRY_RESULT_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result) #define TRY_RESULT_PROMISE(promise_name, name, result) \ @@ -437,6 +446,7 @@ class Status { template class Result { public: + using ValueT = T; Result() : status_(Status::Error<-1>()) { } template , Result>::value, int> = 0> diff --git a/tdutils/test/StealingQueue.cpp b/tdutils/test/StealingQueue.cpp index 6aa2aae6..c5ff8b51 100644 --- a/tdutils/test/StealingQueue.cpp +++ b/tdutils/test/StealingQueue.cpp @@ -28,6 +28,7 @@ #include "td/utils/tests.h" #include "td/utils/benchmark.h" +#include "td/utils/AtomicRead.h" #include "td/utils/StealingQueue.h" #include "td/utils/MpmcQueue.h" @@ -79,6 +80,46 @@ TEST(AtomicRead, simple) { thread.join(); } } +TEST(AtomicRead, simple2) { + td::Stage run; + td::Stage check; + + size_t threads_n = 10; + std::vector threads; + + struct Value { + td::uint64 value = 0; + char str[50] = "0 0 0 0"; + }; + AtomicRead value; + + auto to_str = [](size_t i) { return PSTRING() << i << " " << i << " " << i << " " << i; }; + for (size_t i = 0; i < threads_n; i++) { + threads.push_back(td::thread([&, id = static_cast(i)] { + for (uint64 round = 1; round < 10000; round++) { + if (id == 0) { + } + run.wait(round * threads_n); + if (id == 0) { + auto x = value.lock(); + x->value = round; + auto str = to_str(round); + memcpy(x->str, str.c_str(), str.size() + 1); + } else { + Value x; + value.read(x); + LOG_CHECK(x.value == round || x.value == round - 1) << x.value << " " << round; + CHECK(x.str == to_str(x.value)); + } + check.wait(round * threads_n); + } + })); + } + for (auto &thread : threads) { + thread.join(); + } +} + TEST(StealingQueue, simple) { uint64 sum; std::atomic got_sum; diff --git a/test/test-rocksdb.cpp b/test/test-rocksdb.cpp new file mode 100644 index 00000000..ce13bd92 --- /dev/null +++ b/test/test-rocksdb.cpp @@ -0,0 +1,97 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "td/db/RocksDb.h" +#include "td/utils/OptionsParser.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/path.h" +#include "td/utils/StringBuilder.h" + +#include + +std::string dir = "stress-db"; +int db_n = 20; +int key_n = 1000000; + +std::string get_db_path(int i) { + return PSTRING() << dir << TD_DIR_SLASH << "db-" << i; +} + +void do_create_db() { + td::mkdir(dir).ensure(); + for (int db_i = 0; db_i < db_n; db_i++) { + LOG(ERROR) << "db_i=" << db_i; + auto db = td::RocksDb::open(get_db_path(db_i)).move_as_ok(); + for (int key_i = 0; key_i < key_n; key_i++) { + db.set(PSLICE() << key_i, PSLICE() << key_i); + } + } +} + +void do_load_db() { + static std::vector dbs; + for (int db_i = 0; db_i < db_n; db_i++) { + LOG(ERROR) << "db_i=" << db_i; + auto db = td::RocksDb::open(get_db_path(db_i)).move_as_ok(); + for (int key_i = 0; key_i < key_n; key_i++) { + std::string value; + db.get(PSLICE() << key_i, value).ensure(); + } + dbs.push_back(std::move(db)); + } +} + +int main(int argc, char *argv[]) { + SET_VERBOSITY_LEVEL(verbosity_DEBUG); + td::set_default_failure_signal_handler().ensure(); + + td::OptionsParser p; + p.set_description("test basic adnl functionality"); + p.add_option('h', "help", "prints_help", [&]() { + char b[10240]; + td::StringBuilder sb(td::MutableSlice{b, 10000}); + sb << p; + std::cout << sb.as_cslice().c_str(); + std::exit(2); + return td::Status::OK(); + }); + bool create_db = false; + p.add_option('c', "create", "create test db", [&] { + create_db = true; + return td::Status::OK(); + }); + + auto res = p.run(argc, argv); + LOG_IF(FATAL, res.is_error()) << res.error(); + + if (create_db) { + do_create_db(); + } else { + do_load_db(); + } + return 0; +} diff --git a/third-party/abseil-cpp b/third-party/abseil-cpp index 325fd7b0..df3ea785 160000 --- a/third-party/abseil-cpp +++ b/third-party/abseil-cpp @@ -1 +1 @@ -Subproject commit 325fd7b042ff4ec34f7dd32e602cd81ad0e24b22 +Subproject commit df3ea785d8c30a9503321a3d35ee7d35808f190d diff --git a/third-party/rocksdb b/third-party/rocksdb index d6f2ecf4..0915c99f 160000 --- a/third-party/rocksdb +++ b/third-party/rocksdb @@ -1 +1 @@ -Subproject commit d6f2ecf49c28fee225477d39e2a1535a87919afe +Subproject commit 0915c99f01b46f50af8e02da8b6528156f584b7c diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 64ab40f9..aed8db00 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -25,7 +25,7 @@ keyStoreTypeInMemory = KeyStoreType; config config:string blockchain_name:string use_callbacks_for_network:Bool ignore_cache:Bool = Config; options config:config keystore_type:KeyStoreType = Options; -options.configInfo default_wallet_id:int64 = options.ConfigInfo; +options.configInfo default_wallet_id:int64 default_rwallet_init_public_key:string = options.ConfigInfo; options.info config_info:options.configInfo = options.Info; key public_key:string secret:secureBytes = Key; @@ -55,6 +55,8 @@ raw.message source:accountAddress destination:accountAddress value:int64 fwd_fee raw.transaction utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 storage_fee:int64 other_fee:int64 in_msg:raw.message out_msgs:vector = raw.Transaction; raw.transactions transactions:vector previous_transaction_id:internal.transactionId = raw.Transactions; +pchan.config alice_public_key:string alice_address:accountAddress bob_public_key:string bob_address:accountAddress init_timeout:int32 close_timeout:int32 channel_id:int64 = pchan.Config; + raw.initialAccountState code:bytes data:bytes = InitialAccountState; testGiver.initialAccountState = InitialAccountState; testWallet.initialAccountState public_key:string = InitialAccountState; @@ -62,7 +64,13 @@ wallet.initialAccountState public_key:string = InitialAccountState; wallet.v3.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; wallet.highload.v1.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; wallet.highload.v2.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; + +rwallet.limit seconds:int32 value:int64 = rwallet.Limit; +rwallet.config start_at:int53 limits:vector = rwallet.Config; +rwallet.initialAccountState init_public_key:string public_key:string wallet_id:int64 = InitialAccountState; + dns.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; +pchan.initialAccountState config:pchan.config = InitialAccountState; raw.accountState code:bytes data:bytes frozen_hash:bytes = AccountState; testWallet.accountState seqno:int32 = AccountState; @@ -72,6 +80,13 @@ wallet.highload.v1.accountState wallet_id:int64 seqno:int32 = AccountState; wallet.highload.v2.accountState wallet_id:int64 = AccountState; testGiver.accountState seqno:int32 = AccountState; dns.accountState wallet_id:int64 = AccountState; +rwallet.accountState wallet_id:int64 seqno:int32 unlocked_balance:int64 config:rwallet.config = AccountState; + +pchan.stateInit signed_A:Bool signed_B:Bool min_A:int64 min_B:int64 expire_at:int53 A:int64 B:int64 = pchan.State; +pchan.stateClose signed_A:Bool signed_B:Bool min_A:int64 min_B:int64 expire_at:int53 A:int64 B:int64 = pchan.State; +pchan.statePayout A:int64 B:int64 = pchan.State; + +pchan.accountState config:pchan.config state:pchan.State description:string = AccountState; uninited.accountState frozen_hash:bytes = AccountState; fullAccountState balance:int64 last_transaction_id:internal.transactionId block_id:ton.blockIdExt sync_utime:int53 account_state:AccountState = FullAccountState; @@ -83,7 +98,7 @@ syncStateInProgress from_seqno:int32 to_seqno:int32 current_seqno:int32 = SyncSt // MSG // -msg.dataRaw body:bytes = msg.Data; +msg.dataRaw body:bytes init_state:bytes = msg.Data; msg.dataText text:bytes = msg.Data; msg.dataDecryptedText text:bytes = msg.Data; msg.dataEncryptedText text:bytes = msg.Data; @@ -115,6 +130,21 @@ dns.actionSet entry:dns.entry = dns.Action; dns.resolved entries:vector = dns.Resolved; + +// +// Payment channel +// +pchan.promise signature:bytes promise_A:int64 promise_B:int64 channel_id:int64 = pchan.Promise; + +pchan.actionInit inc_A:int64 inc_B:int64 min_A:int64 min_B:int64 = pchan.Action; +pchan.actionClose extra_A:int64 extra_B:int64 promise:pchan.promise = pchan.Action; +pchan.actionTimeout = pchan.Action; + +// +// Restricted wallet initialization +// +rwallet.actionInit config:rwallet.config = rwallet.Action; + // // Actions // @@ -122,12 +152,14 @@ dns.resolved entries:vector = dns.Resolved; actionNoop = Action; actionMsg messages:vector allow_send_to_uninited:Bool = Action; actionDns actions:vector = Action; +actionPchan action:pchan.Action = Action; +actionRwallet action:rwallet.actionInit = Action; //actionMultisig actions:vector = Action; fees in_fwd_fee:int53 storage_fee:int53 gas_fee:int53 fwd_fee:int53 = Fees; query.fees source_fees:fees destination_fees:vector = query.Fees; // query.emulationResult exit_code:int32 fees:fees = query.EmulationResult; -query.info id:int53 valid_until:int53 body_hash:bytes = query.Info; +query.info id:int53 valid_until:int53 body_hash:bytes body:bytes init_state:bytes = query.Info; tvm.slice bytes:bytes = tvm.Slice; tvm.cell bytes:bytes = tvm.Cell; @@ -218,7 +250,7 @@ sync = ton.BlockIdExt; getAccountAddress initial_account_state:InitialAccountState revision:int32 = AccountAddress; guessAccountRevision initial_account_state:InitialAccountState = AccountRevisionList; getAccountState account_address:accountAddress = FullAccountState; -createQuery private_key:InputKey address:accountAddress timeout:int32 action:Action = query.Info; +createQuery private_key:InputKey address:accountAddress timeout:int32 action:Action initial_account_state:InitialAccountState = query.Info; msg.decrypt input_key:InputKey data:msg.dataEncryptedArray = msg.DataDecryptedArray; msg.decryptWithProof proof:bytes data:msg.dataEncrypted = msg.Data; @@ -238,6 +270,12 @@ smc.runGetMethod id:int53 method:smc.MethodId stack:vector = smc dns.resolve account_address:accountAddress name:string category:int32 ttl:int32 = dns.Resolved; +pchan.signPromise input_key:InputKey promise:pchan.promise = pchan.Promise; +pchan.validatePromise public_key:bytes promise:pchan.promise = Ok; + +pchan.packPromise promise:pchan.promise = Data; +pchan.unpackPromise data:secureBytes = pchan.Promise; + onLiteServerQueryResult id:int64 bytes:bytes = Ok; onLiteServerQueryError id:int64 error:error = Ok; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index f513bc63e21428874d0345dfcf249ae6ef3a860d..0998c4b9b631c41eeb242afeaf39e462fb864317 100644 GIT binary patch delta 3290 zcmcImZERCj7{0e_yMDLZI_Pc7=wL|4MptAA@-eqE0&``Qj}U%{?Ol5rH|yHc-m;M> zgMh@(=yv#Fl$c>m1VYT9@q+|W5~ln>5`!j?5CuX+V@x0;1oe;iJ?EZscQ<}|f4%oT z=lwX(`@HWtW2e|F-?R2RAGWMylyB3B{ZQ?Z_A}quI+Dq3F(Yl2cu9~Kt>|FIZ2MKt zzRlQ?l*DX?F@WaGt%Q#I!_?kd)0Gh=kx#45EU=@Lt5~hqzox&jsx)5dX7#h;Usj^a zrX97}H76amTyi&`2`L^KrUERgo_l!Yu(Gl?o7pAyp?aOKdgoEc$9A$mmd_4|1p&Rx zdaI(`_ET1(#<^k_C}0lib2s1Ej-uw9HT_>NUEs(M^69i7DX~zTxli}_9oxu~ zq@zt-3c zI&JKs?7T!ZDan+`G;W1#@B8i7My+qCJM|6ghJw^c*e4r6AgHwx8#}P3ghDbe=75i* zHa|hE*%egb!`fSj`aPF_1<)5T1|Kwe&{3ZNTx{?QhndF|-&p8l7+6PlT|;^-B&z&G zI8fC&tTnwovG`LfF^vJQVI7xL_K(j?)^Id94s-ww{j_u@mh1ku6T2;h!6YTw?~*8h zHIri?JOxtixOBOy)rjF3^`!OCwkYR`5#f?2>;a7yAH2)Pg~_gVu9XNwT(d@Vi|#q? z-9eUe-W5bBYfQc;nizPd#|PxywsY%zl(ORVaYD1B^l`$Tc&^x+7L)OQVIN8?@Sv}} zQ4({^7c2+&YD0C-X*^=f8PK*6RI^uO-l_U<&kV;v~;g;Ko4 zZ{Y`F*a3_>^kdI_G`_@O#hDiHC`CD!ACE)ycu{}|{-!VyVogDfW_+5))tn-GhMMnV zp_E=6W{GMn6sU*!TAjr$Q3#P^bY*sB8PJoYZiiZ+wF7fg!Y;m;mf{NLaSfi(QFvPW zTRMs0*$@c5`KZb3qRkDB*Ckg$E+dc`{Oc7?8pKR8-hnPSd{~NLX+X>1;v1f@lC*{z_p;%JZ_jZyg?!@)Lod;mRDjAAnrYXK-_9+d&(3 z@<82SjXLT!aX`oxB^nic(aLLWRreJD{){M(4F;5eN2M%7kGgo7rh=NT59<65qU2Ml zyih1G=!M0c!O@%TjiY++e1jEBY};JB4VOkDDWT(TpTnL6RN_>1b)L@zBPS!I<7i~Q zJiDEyHb?>Th7XWc-JoTJK&LL7xghpsc&V5d&{dn)VpcZds;PdVoL;kKT1aq50+NuA zZHvx139Oj@Ko*$dYGh0hQoedR81z|<*Ft|I_}6IAaz+kpH48&IF)zeoO_bJ)2dJwC!r9E`M(ocsZC_P`=f;lwQ(XbexFP+rksdQh7Pzg zZO{$FWl>GXF1cxNOM@P~KW!$?^b!CYb@`{fu#}k!I`FF;*4Ai}vB*nMOI*xKV;UY_ z1r)c~?CuVe3A>>?NQAQHM0alWMa$|R6yD_cWWVzKWtcrD*WXi@ zXr{r3uCxXGb@7HYC7#@DWe3o%Xsja~o5@m(ySBHl-F@YAWn>mJs=TmMS8X_~HMnyr z<*X&cV6Qy*nC*D8$j1>F855?pxnj)@@juu{BEA3s delta 1052 zcmX?eoN>V_M&3uW^{p77z+)nBv#8pd3LYNE zCZFNdVBuc5A#U;y&O4hUxaTl3=1%^~^O*%?>*P1QB@jjmp9X|6fzJTKIKrobCZfO( zRpY}S0+Cw64^<2oF%U4|1etvEn2-5;#)5*b$eB%t!?x`pNDLa33|Hm2Vgkyy@o^?5mt^MW`4*=$Fo5JIw+L6TfEbfR zWWe-f1rZl8W3z~;2^%ZOxeSvH1LP;?Nc*sW0(tTt=>P~rK*m40>XfcgvgOU}bwh8$EtAN5A#0zu}%Ok$aA8>c<_fLDM7ateW^8nCGm-T^G& zK;FE@KY$6$+7#3S$wHGSgatrsd=e(Z$cgaRL^1Kn3K1d@?^r}=;F6jVQ84*P)C)+) e+Yqe-W^8^Ct-%NpT3}=_*(NpzBD^M6feiqg?t+s5 diff --git a/tonlib/test/online.cpp b/tonlib/test/online.cpp index 6aba742d..acafc199 100644 --- a/tonlib/test/online.cpp +++ b/tonlib/test/online.cpp @@ -121,11 +121,12 @@ struct TransactionId { }; struct AccountState { - enum Type { Empty, Wallet, Dns, Unknown } type{Empty}; + enum Type { Empty, Wallet, Dns, Pchan, Unknown } type{Empty}; td::int64 sync_utime{-1}; td::int64 balance{-1}; TransactionId last_transaction_id; std::string address; + tonlib_api::object_ptr state; bool is_inited() const { return type != Empty; @@ -187,6 +188,10 @@ AccountState get_account_state(Client& client, std::string address) { case tonlib_api::dns_accountState::ID: res.type = AccountState::Dns; break; + case tonlib_api::pchan_accountState::ID: + res.type = AccountState::Pchan; + res.state = std::move(state); + break; default: res.type = AccountState::Unknown; break; @@ -228,15 +233,38 @@ struct QueryInfo { std::string body_hash; }; +struct Message { + bool encrypted = false; + td::optional text; + td::optional raw; + td::optional init_state; + static Message create_text(std::string text, bool encrypted) { + Message res; + res.text = text; + res.encrypted = encrypted; + return res; + } + static Message create_raw(std::string raw, std::string init_state) { + Message res; + res.raw = raw; + res.init_state = init_state; + return res; + } +}; + td::Result create_send_grams_query(Client& client, const Wallet& source, std::string destination, - td::int64 amount, bool encrypted, std::string message, bool force = false, - int timeout = 0, bool fake = false) { + td::int64 amount, Message message, bool force = false, int timeout = 0, + bool fake = false) { std::vector> msgs; tonlib_api::object_ptr data; - if (encrypted) { - data = tonlib_api::make_object(std::move(message)); + if (message.text) { + if (message.encrypted) { + data = tonlib_api::make_object(message.text.unwrap()); + } else { + data = tonlib_api::make_object(message.text.unwrap()); + } } else { - data = tonlib_api::make_object(std::move(message)); + data = tonlib_api::make_object(message.raw.unwrap(), message.init_state.unwrap()); } msgs.push_back(tonlib_api::make_object( tonlib_api::make_object(destination), "", amount, std::move(data))); @@ -244,7 +272,7 @@ td::Result create_send_grams_query(Client& client, const Wallet& source auto r_id = sync_send(client, tonlib_api::make_object( fake ? source.key.get_fake_input_key() : source.key.get_input_key(), source.get_address(), - timeout, tonlib_api::make_object(std::move(msgs), force))); + timeout, tonlib_api::make_object(std::move(msgs), force), nullptr)); TRY_RESULT(id, std::move(r_id)); return QueryId{id->id_}; } @@ -253,7 +281,7 @@ td::Result create_update_dns_query(Client& client, const Wallet& dns, std::vector> entries) { using namespace ton::tonlib_api; auto r_id = sync_send(client, make_object(dns.key.get_input_key(), dns.get_address(), 60, - make_object(std::move(entries)))); + make_object(std::move(entries)), nullptr)); TRY_RESULT(id, std::move(r_id)); return QueryId{id->id_}; } @@ -289,7 +317,7 @@ td::Result wait_state_change(Client& client, const AccountState& o while (true) { auto new_state = get_account_state(client, old_state.address); if (new_state.last_transaction_id.lt != old_state.last_transaction_id.lt) { - return new_state; + return std::move(new_state); } if (valid_until != 0 && new_state.sync_utime >= valid_until) { return td::Status::Error("valid_until expired"); @@ -325,18 +353,19 @@ td::Status transfer_grams(Client& client, const Wallet& wallet, std::string addr LOG(INFO) << "Transfer: create query " << (double)amount / Gramm << " from " << wallet.address << " to " << address; bool encrypt = true; - auto r_query_id = create_send_grams_query(client, wallet, address, amount, encrypt, message, fast); + auto r_query_id = + create_send_grams_query(client, wallet, address, amount, Message::create_text(message, encrypt), fast); if (r_query_id.is_error()) { LOG(INFO) << "Send query WITHOUT message encryption " << r_query_id.error(); encrypt = false; - r_query_id = create_send_grams_query(client, wallet, address, amount, encrypt, message, fast); + r_query_id = create_send_grams_query(client, wallet, address, amount, Message::create_text(message, encrypt), fast); } else { LOG(INFO) << "Send query WITH message encryption"; } if (r_query_id.is_error() && td::begins_with(r_query_id.error().message(), "DANGEROUS_TRANSACTION")) { ASSERT_TRUE(dst_state.type == AccountState::Empty); LOG(INFO) << "Transfer: recreate query due to DANGEROUS_TRANSACTION error"; - r_query_id = create_send_grams_query(client, wallet, address, amount, false, message, true); + r_query_id = create_send_grams_query(client, wallet, address, amount, Message::create_text(message, encrypt), true); } r_query_id.ensure(); @@ -458,8 +487,9 @@ Wallet create_empty_dns(Client& client) { void test_estimate_fees_without_key(Client& client, const Wallet& wallet_a, const Wallet& wallet_b) { LOG(ERROR) << " SUBTEST: estimate fees without key"; { - auto query_id = - create_send_grams_query(client, wallet_a, wallet_b.address, 0, false, "???", true, 0, true).move_as_ok(); + auto query_id = create_send_grams_query(client, wallet_a, wallet_b.address, 0, Message::create_text("???", false), + true, 0, true) + .move_as_ok(); auto fees1 = query_estimate_fees(client, query_id, false); auto fees2 = query_estimate_fees(client, query_id, true); LOG(INFO) << "Fee without ignore_chksig\t" << fees1; @@ -564,6 +594,141 @@ void dns_resolve(Client& client, const Wallet& dns, std::string name) { LOG(INFO) << "OK"; } +void test_paychan(Client& client, const Wallet& giver_wallet) { + LOG(INFO) << "Start test paychan"; + auto alice = create_empty_wallet(client); + auto bob = create_empty_wallet(client); + + using namespace ton::tonlib_api; + int init_timeout = 10; + int close_timeout = 10; + int64 channel_id = static_cast(td::Random::fast_uint64()); + + auto get_initial_state = [&] { + return make_object(make_object(alice.key.public_key, alice.get_address(), + bob.key.public_key, bob.get_address(), + init_timeout, close_timeout, channel_id)); + }; + auto account_address = sync_send(client, make_object(get_initial_state(), -1)) + .move_as_ok() + ->account_address_; + auto get_account_address = [&] { return make_object(account_address); }; + + //pchan.actionInit inc_A:int64 inc_B:int64 min_A:int64 min_B:int64 = pchan.Action; + //pchan.actionClose extra_A:int64 extra_B:int64 promise:pchan.promise = pchan.Action; + //pchan.actionTimeout = pchan.Action; + + auto create_init_query = [&](auto& wallet, td::int64 inc_A, td::int64 inc_B) { + auto action = make_object(make_object(inc_A, inc_B, 0, 0)); + auto r_id = sync_send(client, make_object(wallet.key.get_input_key(), get_account_address(), 60, + std::move(action), get_initial_state())); + r_id.ensure(); + return r_id.move_as_ok(); + }; + + auto send_query_via = [&](auto& wallet, auto query_info, auto destination, td::int64 value) { + auto transfer_id = + create_send_grams_query(client, wallet, std::move(destination), value, + Message::create_raw(query_info->body_, query_info->init_state_), true, 60) + .move_as_ok(); + ::query_send(client, transfer_id); + }; + send_query_via(giver_wallet, create_init_query(alice, 1 * Gramm, 0), account_address, 11 * Gramm / 10); + send_query_via(giver_wallet, create_init_query(bob, 0, 1 * Gramm), account_address, 11 * Gramm / 10); + + LOG(INFO) << "Wait till pchan " << account_address << " is inited "; + td::optional now; + while (true) { + client.receive(1); + auto state = get_account_state(client, account_address); + if (!now) { + now = state.sync_utime; + } + CHECK(now.value() + 60 > state.sync_utime); + if (state.type != ::AccountState::Pchan) { + continue; + } + auto pchan_state = tonlib_api::move_object_as(state.state->account_state_); + if (pchan_state->state_->get_id() != tonlib_api::pchan_stateClose::ID) { + continue; + } + LOG(INFO) << "Account type: " << state.type << " " << state.sync_utime << "\n" << to_string(pchan_state->state_); + break; + } + + auto create_close_query = [&](auto& x_wallet, auto& y_wallet, td::int64 A, td::int64 B) { + auto p = sync_send(client, make_object(y_wallet.key.get_input_key(), + make_object("", A, B, channel_id))) + .move_as_ok(); + auto action = make_object(make_object(0, 0, std::move(p))); + auto r_id = sync_send(client, make_object(x_wallet.key.get_input_key(), get_account_address(), 60, + std::move(action), get_initial_state())); + r_id.ensure(); + return r_id.move_as_ok(); + }; + + //auto send_query_ext = [&](auto query_info) { ::query_send(client, QueryId{query_info->id_}); }; + + send_query_via(giver_wallet, create_close_query(alice, bob, 0, 10 * Gramm), account_address, Gramm / 10); + send_query_via(giver_wallet, create_close_query(bob, alice, 11 * Gramm, 0), account_address, Gramm / 10); + + LOG(INFO) << "Wait till pchan " << account_address << " is closed "; + now = {}; + int64 payout_A = 0; + int64 payout_B = 0; + while (true) { + client.receive(1); + auto state = get_account_state(client, account_address); + if (!now) { + now = state.sync_utime; + } + CHECK(now.value() + 60 > state.sync_utime); + if (state.type != ::AccountState::Pchan) { + continue; + } + auto pchan_state = tonlib_api::move_object_as(state.state->account_state_); + if (pchan_state->state_->get_id() != tonlib_api::pchan_statePayout::ID) { + continue; + } + LOG(INFO) << "Account type: " << state.type << " " << state.sync_utime << "\n" << to_string(pchan_state->state_); + auto payout = tonlib_api::move_object_as(pchan_state->state_); + payout_A = payout->A_; + payout_B = payout->B_; + break; + } + + LOG(INFO) << "Wait till Alice has its share"; + now = {}; + while (payout_A != 0) { + auto state = get_account_state(client, alice.address); + if (!now) { + now = state.sync_utime; + } + CHECK(now.value() + 60 > state.sync_utime); + if (state.balance > 0) { + ASSERT_EQ(payout_A, state.balance); + LOG(INFO) << "Alice got: " << state.balance; + break; + } + client.receive(1); + } + LOG(INFO) << "Wait till Bob has its share"; + now = {}; + while (payout_B != 0) { + auto state = get_account_state(client, bob.address); + if (!now) { + now = state.sync_utime; + } + CHECK(now.value() + 60 > state.sync_utime); + if (state.balance > 0) { + ASSERT_EQ(payout_B, state.balance); + LOG(INFO) << "Bob got: " << state.balance; + break; + } + client.receive(1); + } +} + void test_dns(Client& client, const Wallet& giver_wallet) { auto A = create_empty_dns(client); auto A_B = create_empty_dns(client); @@ -669,6 +834,7 @@ int main(int argc, char* argv[]) { // give wallet with some test grams to run test auto giver_wallet = import_wallet_from_pkey(client, giver_key_str, giver_key_pwd); + test_paychan(client, giver_wallet); test_dns(client, giver_wallet); test_back_and_forth_transfer(client, giver_wallet, false); test_back_and_forth_transfer(client, giver_wallet, true); diff --git a/tonlib/tonlib/Client.cpp b/tonlib/tonlib/Client.cpp index b4839ca6..056aa0db 100644 --- a/tonlib/tonlib/Client.cpp +++ b/tonlib/tonlib/Client.cpp @@ -101,6 +101,7 @@ class Client::Impl final { scheduler_.run_in_context_external([] { td::actor::SchedulerContext::get()->stop(); }); LOG(ERROR) << "join"; scheduler_thread_.join(); + LOG(ERROR) << "join - done"; } private: diff --git a/tonlib/tonlib/Config.cpp b/tonlib/tonlib/Config.cpp index 8d48327e..f4a6dc80 100644 --- a/tonlib/tonlib/Config.cpp +++ b/tonlib/tonlib/Config.cpp @@ -121,6 +121,26 @@ td::Result Config::parse(std::string str) { res.init_block_id = init_block_id; } + auto r_hardforks = td::get_json_object_field(validator, "hardforks", td::JsonValue::Type::Array, false); + if (r_hardforks.is_ok()) { + auto hardforks_obj = r_hardforks.move_as_ok(); + auto &hardforks = hardforks_obj.get_array(); + for (auto &fork : hardforks) { + if (fork.type() != td::JsonValue::Type::Object) { + return td::Status::Error("Invalid config (8)"); + } + TRY_RESULT(fork_block, parse_block_id_ext(fork.get_object())); + res.hardforks.push_back(std::move(fork_block)); + } + } + + for (auto hardfork : res.hardforks) { + if (!res.init_block_id.is_valid() || hardfork.seqno() > res.init_block_id.seqno()) { + LOG(INFO) << "Replace init_block with hardfork: " << res.init_block_id.to_str() << " -> " << hardfork.to_str(); + res.init_block_id = hardfork; + } + } + return res; } } // namespace tonlib diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 01831542..3902c341 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -29,7 +29,9 @@ struct Config { }; ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; + std::vector hardforks; std::vector lite_clients; + std::string name; static td::Result parse(std::string str); }; } // namespace tonlib diff --git a/tonlib/tonlib/LastBlock.h b/tonlib/tonlib/LastBlock.h index 9ee1e699..df6e192c 100644 --- a/tonlib/tonlib/LastBlock.h +++ b/tonlib/tonlib/LastBlock.h @@ -93,9 +93,10 @@ struct LastBlockState { ton::BlockIdExt last_block_id; td::int64 utime{0}; ton::BlockIdExt init_block_id; + td::int32 vert_seqno{0}; static constexpr td::int32 magic = 0xa7f171a4; - enum Version { None = 0, Magic, InitBlock, Next }; + enum Version { None = 0, Magic, InitBlock, VertSeqno, Next }; static constexpr td::int32 version = Version::Next - 1; template @@ -110,6 +111,7 @@ struct LastBlockState { store(last_block_id, storer); store(utime, storer); store(init_block_id, storer); + store(vert_seqno, storer); } template @@ -130,6 +132,9 @@ struct LastBlockState { if (version >= InitBlock) { parse(init_block_id, parser); } + if (version >= VertSeqno) { + parse(vert_seqno, parser); + } } }; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 5ad49560..f7e9223c 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -36,6 +36,7 @@ #include "smc-envelope/HighloadWallet.h" #include "smc-envelope/HighloadWalletV2.h" #include "smc-envelope/TestGiver.h" +#include "smc-envelope/PaymentChannel.h" #include "smc-envelope/SmartContractCode.h" #include "auto/tl/tonlib_api.hpp" @@ -106,7 +107,8 @@ auto to_tonlib_api(const ton::BlockIdExt& blk) { } tonlib_api::object_ptr to_tonlib_api(const TonlibClient::FullConfig& full_config) { - return tonlib_api::make_object(full_config.wallet_id); + return tonlib_api::make_object(full_config.wallet_id, + full_config.rwallet_init_public_key); } class TonlibQueryActor : public td::actor::Actor { @@ -172,9 +174,69 @@ tonlib_api::object_ptr to_transaction_id(con } std::string to_bytes(td::Ref cell) { + if (cell.is_null()) { + return ""; + } return vm::std_boc_serialize(cell, vm::BagOfCells::Mode::WithCRC32C).move_as_ok().as_slice().str(); } +td::Result get_public_key(td::Slice public_key) { + TRY_RESULT_PREFIX(address, block::PublicKey::parse(public_key), TonlibError::InvalidPublicKey()); + return address; +} + +td::Result get_account_address(td::Slice account_address) { + TRY_RESULT_PREFIX(address, block::StdAddress::parse(account_address), TonlibError::InvalidAccountAddress()); + return address; +} + +td::Result public_key_from_bytes(td::Slice bytes) { + TRY_RESULT_PREFIX(key_bytes, block::PublicKey::from_bytes(bytes), TonlibError::Internal()); + return key_bytes; +} + +td::Result to_init_data(const tonlib_api::rwallet_initialAccountState& rwallet_state) { + TRY_RESULT(init_key_bytes, get_public_key(rwallet_state.init_public_key_)); + TRY_RESULT(key_bytes, get_public_key(rwallet_state.public_key_)); + ton::RestrictedWallet::InitData init_data; + init_data.init_key = td::SecureString(init_key_bytes.key); + init_data.main_key = td::SecureString(key_bytes.key); + init_data.wallet_id = static_cast(rwallet_state.wallet_id_); + return std::move(init_data); +} + +td::Result to_pchan_config(const tonlib_api::pchan_initialAccountState& pchan_state) { + ton::pchan::Config config; + if (!pchan_state.config_) { + return TonlibError::EmptyField("config"); + } + TRY_RESULT_PREFIX(a_key, get_public_key(pchan_state.config_->alice_public_key_), + TonlibError::InvalidField("alice_public_key", "")); + config.a_key = td::SecureString(a_key.key); + TRY_RESULT_PREFIX(b_key, get_public_key(pchan_state.config_->bob_public_key_), + TonlibError::InvalidField("bob_public_key", "")); + config.b_key = td::SecureString(b_key.key); + + if (!pchan_state.config_->alice_address_) { + return TonlibError::EmptyField("config.alice_address"); + } + TRY_RESULT_PREFIX(a_addr, get_account_address(pchan_state.config_->alice_address_->account_address_), + TonlibError::InvalidField("alice_address", "")); + config.a_addr = std::move(a_addr); + + if (!pchan_state.config_->bob_address_) { + return TonlibError::EmptyField("config.bob_address"); + } + TRY_RESULT_PREFIX(b_addr, get_account_address(pchan_state.config_->bob_address_->account_address_), + TonlibError::InvalidField("bob_address", "")); + config.b_addr = std::move(b_addr); + + config.channel_id = pchan_state.config_->channel_id_; + config.init_timeout = pchan_state.config_->init_timeout_; + config.close_timeout = pchan_state.config_->close_timeout_; + return std::move(config); +} + class AccountState { public: AccountState(block::StdAddress address, RawAccountState&& raw, td::uint32 wallet_id) @@ -259,6 +321,55 @@ class AccountState { TRY_RESULT(wallet_id, wallet.get_wallet_id()); return tonlib_api::make_object(static_cast(wallet_id)); } + td::Result> to_rwallet_accountState() const { + if (wallet_type_ != RestrictedWallet) { + return TonlibError::AccountTypeUnexpected("RestrictedWallet"); + } + auto wallet = ton::RestrictedWallet::create(get_smc_state()); + TRY_RESULT(seqno, wallet->get_seqno()); + TRY_RESULT(wallet_id, wallet->get_wallet_id()); + TRY_RESULT(balance, wallet->get_balance(raw_.balance, raw_.info.gen_utime)); + TRY_RESULT(config, wallet->get_config()); + + auto api_config = tonlib_api::make_object(); + api_config->start_at_ = config.start_at; + for (auto& limit : config.limits) { + api_config->limits_.push_back(tonlib_api::make_object(limit.first, limit.second)); + } + + return tonlib_api::make_object(wallet_id, seqno, balance, std::move(api_config)); + } + td::Result> to_payment_channel_accountState() const { + if (wallet_type_ != PaymentChannel) { + return TonlibError::AccountTypeUnexpected("PaymentChannel"); + } + auto pchan = ton::PaymentChannel::create(get_smc_state()); + TRY_RESULT(info, pchan->get_info()); + TRY_RESULT(a_key, public_key_from_bytes(info.config.a_key)); + TRY_RESULT(b_key, public_key_from_bytes(info.config.b_key)); + + tonlib_api::object_ptr tl_state; + info.state.visit(td::overloaded( + [&](const ton::pchan::StateInit& state) { + tl_state = tonlib_api::make_object( + state.signed_A, state.signed_B, state.min_A, state.min_B, state.A, state.B, state.expire_at); + }, + [&](const ton::pchan::StateClose& state) { + tl_state = tonlib_api::make_object( + state.signed_A, state.signed_B, state.promise_A, state.promise_B, state.A, state.B, state.expire_at); + }, + [&](const ton::pchan::StatePayout& state) { + tl_state = tonlib_api::make_object(state.A, state.B); + })); + + using tonlib_api::make_object; + return tonlib_api::make_object( + tonlib_api::make_object( + a_key.serialize(true), make_object(info.config.a_addr.rserialize(true)), + b_key.serialize(true), make_object(info.config.b_addr.rserialize(true)), + info.config.init_timeout, info.config.close_timeout, info.config.channel_id), + std::move(tl_state), info.description); + } td::Result> to_testGiver_accountState() const { if (wallet_type_ != Giver) { @@ -298,11 +409,14 @@ class AccountState { return f(to_wallet_highload_v1_accountState()); case HighloadWalletV2: return f(to_wallet_highload_v2_accountState()); + case RestrictedWallet: + return f(to_rwallet_accountState()); case ManualDns: return f(to_dns_accountState()); - default: - UNREACHABLE(); + case PaymentChannel: + return f(to_payment_channel_accountState()); } + UNREACHABLE(); } td::Result> to_fullAccountState() const { @@ -321,7 +435,9 @@ class AccountState { WalletV3, HighloadWalletV1, HighloadWalletV2, - ManualDns + ManualDns, + PaymentChannel, + RestrictedWallet }; WalletType get_wallet_type() const { return wallet_type_; @@ -331,6 +447,7 @@ class AccountState { case AccountState::Empty: case AccountState::Unknown: case AccountState::ManualDns: + case AccountState::PaymentChannel: return false; case AccountState::Giver: case AccountState::SimpleWallet: @@ -338,6 +455,7 @@ class AccountState { case AccountState::WalletV3: case AccountState::HighloadWalletV1: case AccountState::HighloadWalletV2: + case AccountState::RestrictedWallet: return true; } UNREACHABLE(); @@ -348,6 +466,7 @@ class AccountState { case AccountState::Empty: case AccountState::Unknown: case AccountState::ManualDns: + case AccountState::PaymentChannel: return {}; case AccountState::Giver: return td::make_unique(get_smc_state()); @@ -361,6 +480,8 @@ class AccountState { return td::make_unique(get_smc_state()); case AccountState::HighloadWalletV2: return td::make_unique(get_smc_state()); + case AccountState::RestrictedWallet: + return td::make_unique(get_smc_state()); } UNREACHABLE(); return {}; @@ -393,6 +514,50 @@ class AccountState { return raw_; } + WalletType guess_type_by_init_state(tonlib_api::InitialAccountState& initial_account_state) { + if (wallet_type_ != WalletType::Empty) { + return wallet_type_; + } + downcast_call( + initial_account_state, + td::overloaded( + [](auto& x) {}, + [&](tonlib_api::rwallet_initialAccountState& rwallet) { + for (auto revision : ton::SmartContractCode::get_revisions(ton::SmartContractCode::RestrictedWallet)) { + auto r_init_data = to_init_data(rwallet); + if (r_init_data.is_error()) { + continue; + } + auto wallet = ton::RestrictedWallet::create(r_init_data.move_as_ok(), revision); + if (!(wallet->get_address() == address_)) { + continue; + } + wallet_type_ = WalletType::RestrictedWallet; + wallet_revision_ = revision; + set_new_state(wallet->get_state()); + break; + } + }, + [&](tonlib_api::pchan_initialAccountState& pchan) { + for (auto revision : ton::SmartContractCode::get_revisions(ton::SmartContractCode::PaymentChannel)) { + auto r_conf = to_pchan_config(pchan); + if (r_conf.is_error()) { + continue; + } + auto conf = r_conf.move_as_ok(); + auto wallet = ton::PaymentChannel::create(conf, -1); + if (!(wallet->get_address() == address_)) { + continue; + } + wallet_type_ = WalletType::PaymentChannel; + wallet_revision_ = revision; + set_new_state(wallet->get_state()); + break; + } + })); + return wallet_type_; + } + WalletType guess_type_by_public_key(td::Ed25519::PublicKey& key) { if (wallet_type_ != WalletType::Empty) { return wallet_type_; @@ -509,6 +674,18 @@ class AccountState { wallet_revision_ = o_revision.value(); return wallet_type_; } + o_revision = ton::PaymentChannel::guess_revision(code_hash); + if (o_revision) { + wallet_type_ = WalletType::PaymentChannel; + wallet_revision_ = o_revision.value(); + return wallet_type_; + } + o_revision = ton::RestrictedWallet::guess_revision(code_hash); + if (o_revision) { + wallet_type_ = WalletType::RestrictedWallet; + wallet_revision_ = o_revision.value(); + return wallet_type_; + } if (code_hash == ton::TestGiver::get_init_code_hash()) { wallet_type_ = WalletType::Giver; @@ -516,8 +693,6 @@ class AccountState { wallet_type_ = WalletType::SimpleWallet; } else if (code_hash == ton::Wallet::get_init_code_hash()) { wallet_type_ = WalletType::Wallet; - } else if (code_hash == ton::HighloadWallet::get_init_code_hash()) { - wallet_type_ = WalletType::HighloadWalletV1; } else { LOG(WARNING) << "Unknown code hash: " << td::base64_encode(code_hash.as_slice()); wallet_type_ = WalletType::Unknown; @@ -545,6 +720,12 @@ class Query { td::Ref get_message() const { return raw_.message; } + td::Ref get_message_body() const { + return raw_.message_body; + } + td::Ref get_init_state() const { + return raw_.new_state; + } vm::CellHash get_body_hash() const { return raw_.message_body->get_hash(); @@ -722,11 +903,8 @@ class Query { raw_.message_body, ton::SmartContract::Args().set_limits(gas_limits).set_ignore_chksig(ignore_chksig)); td::int64 fwd_fee = 0; if (res.success) { - //std::cerr << "new smart contract data: "; - //load_cell_slice(res.new_state.data).print_rec(std::cerr); - //std::cerr << "output actions: "; - //int out_act_num = output_actions_count(res.actions); - //block::gen::OutList{out_act_num}.print_ref(std::cerr, res.actions); + LOG(DEBUG) << "output actions:\n" + << block::gen::OutList{output_actions_count(res.actions)}.as_string_ref(res.actions); TRY_RESULT_ASSIGN(fwd_fee, calc_fwd_fees(res.actions, msg_prices, is_masterchain)); } @@ -1402,11 +1580,6 @@ tonlib_api::object_ptr TonlibClient::do_static_request(const return tonlib_api::make_object(); } -td::Result get_public_key(td::Slice public_key) { - TRY_RESULT_PREFIX(address, block::PublicKey::parse(public_key), TonlibError::InvalidPublicKey()); - return address; -} - td::Result get_account_address(const tonlib_api::raw_initialAccountState& raw_state, td::int32 revision) { TRY_RESULT_PREFIX(code, vm::std_boc_deserialize(raw_state.code_), TonlibError::InvalidBagOfCells("raw_state.code")); @@ -1467,9 +1640,15 @@ td::Result get_account_address(const tonlib_api::dns_initialA return ton::ManualDns::create(key, static_cast(dns_state.wallet_id_), revision)->get_address(); } -td::Result get_account_address(td::Slice account_address) { - TRY_RESULT_PREFIX(address, block::StdAddress::parse(account_address), TonlibError::InvalidAccountAddress()); - return address; +td::Result get_account_address(const tonlib_api::pchan_initialAccountState& pchan_state, + td::int32 revision) { + TRY_RESULT(config, to_pchan_config(pchan_state)); + return ton::PaymentChannel::create(config, revision)->get_address(); +} +td::Result get_account_address(const tonlib_api::rwallet_initialAccountState& rwallet_state, + td::int32 revision) { + TRY_RESULT(init_data, to_init_data(rwallet_state)); + return ton::RestrictedWallet::create(init_data, revision)->get_address(); } td::Result get_adnl_address(td::Slice adnl_address) { @@ -1493,6 +1672,8 @@ static td::optional get_wallet_type(tonlib_api::In [](const tonlib_api::wallet_highload_v2_initialAccountState&) { return ton::SmartContractCode::HighloadWalletV2; }, + [](const tonlib_api::rwallet_initialAccountState&) { return ton::SmartContractCode::RestrictedWallet; }, + [](const tonlib_api::pchan_initialAccountState&) { return ton::SmartContractCode::PaymentChannel; }, [](const tonlib_api::dns_initialAccountState&) { return ton::SmartContractCode::ManualDns; })); } @@ -1555,7 +1736,18 @@ td::Status TonlibClient::do_request(const tonlib_api::guessAccountRevision& requ td::Promise> promise_; size_t left_{0}; - std::vector res; + struct Item { + bool is_inited; + td::int64 balance; + int revision; + auto key() const { + return std::make_tuple(is_inited, balance, revision); + } + bool operator<(const Item& other) const { + return key() > other.key(); + } + }; + std::vector res; void start_up() { left_ += addresses_.size(); @@ -1565,12 +1757,24 @@ td::Status TonlibClient::do_request(const tonlib_api::guessAccountRevision& requ } } void on_account_state(int revision, td::Result> r_state) { - if (r_state.is_ok() && r_state.ok()->get_wallet_type() != AccountState::WalletType::Empty) { - res.push_back(revision); + SCOPE_EXIT { + on_account_state_finish(); + }; + if (!r_state.is_ok()) { + return; } + auto state = r_state.move_as_ok(); + if (state->get_balance() < 0) { + return; + } + + res.push_back({state->get_wallet_type() != AccountState::WalletType::Empty, state->get_balance(), revision}); + } + void on_account_state_finish() { left_--; if (left_ == 0) { - promise_.set_value(std::move(res)); + std::sort(res.begin(), res.end()); + promise_.set_value(td::transform(std::move(res), [](auto x) { return x.revision; })); stop(); } } @@ -1653,6 +1857,7 @@ class MasterConfig { public: void add_config(std::string name, std::string json) { auto config = std::make_shared(Config::parse(json).move_as_ok()); + config->name = name; if (!name.empty()) { by_name_[name] = config; } @@ -1701,14 +1906,6 @@ const MasterConfig& get_default_master_config() { })abc"); res.add_config("testnet2", R"abc({ "liteservers": [ - { - "ip": 1137658550, - "port": 4924, - "id": { - "@type": "pub.ed25519", - "key": "peJTw/arlRfssgTuf9BMypJzqOi7SXEqSPSWiEw2U1M=" - } - } ], "validator": { "@type": "validator.config.global", @@ -1721,6 +1918,20 @@ const MasterConfig& get_default_master_config() { }, "init_block": {"workchain":-1,"shard":-9223372036854775808,"seqno":2908451,"root_hash":"5+7X1QHVUBFLFMwa/yd/2fGzt2KeQtwr+o6UUFOQ7Qc=","file_hash":"gmiUgrtAbvEJZYDEkcbeNOhGPS3g+qCepSOEBFLZFzk="} } +})abc"); + res.add_config("mainnet", R"abc({ + "liteservers": [ + ], + "validator": { + "@type": "validator.config.global", + "zero_state": { + "workchain": -1, + "shard": -9223372036854775808, + "seqno": 0, + "root_hash": "KbTmuSarbve4ce9SET3BRyDnlPZr4GMdWswt1nDQAl4=", + "file_hash": "yRCjDRa-ChWiAGf5n3b25U17aqKzVL13C2Tzz8sqtJA=" + } + } })abc"); return res; }(); @@ -1740,7 +1951,6 @@ td::Result TonlibClient::validate_config(tonlib_api::o if (new_config.lite_clients.empty() && !config->use_callbacks_for_network_) { return TonlibError::InvalidConfig("no lite clients"); } - td::optional o_master_config; std::string last_state_key; if (config->blockchain_name_.empty()) { @@ -1755,6 +1965,12 @@ td::Result TonlibClient::validate_config(tonlib_api::o return TonlibError::InvalidConfig("zero_state differs from embedded zero_state"); } + if (o_master_config && o_master_config.value().hardforks != new_config.hardforks) { + return TonlibError::InvalidConfig("hardforks differs from embedded hardforks"); + } + + int vert_seqno = static_cast(new_config.hardforks.size()); + LastBlockState state; td::Result r_state; if (!config->ignore_cache_) { @@ -1773,6 +1989,25 @@ td::Result TonlibClient::validate_config(tonlib_api::o LOG(ERROR) << state.zero_state_id.to_str() << " " << zero_state.to_str(); return TonlibError::InvalidConfig("zero_state differs from cached zero_state"); } + if (state.vert_seqno > vert_seqno) { + LOG(ERROR) << "Stored vert_seqno is bigger than one in config: " << state.vert_seqno << " vs " << vert_seqno; + return TonlibError::InvalidConfig("vert_seqno in cached state is bigger"); + } + if (state.vert_seqno < vert_seqno) { + state.zero_state_id = zero_state; + state.last_block_id = new_config.zero_state_id; + state.last_key_block_id = new_config.zero_state_id; + state.init_block_id = ton::BlockIdExt{}; + LOG(WARNING) << "Drop cached state - vert_seqno is smaller than in config"; + } + } + state.vert_seqno = vert_seqno; + + //TODO: this could be useful to override master config + if (false && new_config.init_block_id.is_valid() && + state.last_key_block_id.id.seqno < new_config.init_block_id.id.seqno) { + state.last_key_block_id = new_config.init_block_id; + LOG(INFO) << "Use init block from USER config: " << new_config.init_block_id.to_str(); } if (o_master_config) { @@ -1782,12 +2017,23 @@ td::Result TonlibClient::validate_config(tonlib_api::o state.last_key_block_id = master_config.init_block_id; LOG(INFO) << "Use init block from MASTER config: " << master_config.init_block_id.to_str(); } + if (!master_config.name.empty()) { + if (new_config.name != master_config.name) { + LOG(INFO) << "Use blockchain name from MASTER config: '" << master_config.name << "' (was '" << new_config.name + << "')"; + new_config.name = master_config.name; + } + } } FullConfig res; res.config = std::move(new_config); res.use_callbacks_for_network = config->use_callbacks_for_network_; res.wallet_id = td::as(res.config.zero_state_id.root_hash.as_slice().data()); + if (res.config.name.empty()) { // TODO == "mainnet" + res.wallet_id = 0x4BA92D89; + } + res.rwallet_init_public_key = "Puasxr0QfFZZnYISRphVse7XHKfW7pZU5SJarVHXvQ+rpzkD"; res.last_state_key = std::move(last_state_key); res.last_state = std::move(state); @@ -1906,7 +2152,7 @@ struct ToRawTransactions { } } if (!data) { - data = tonlib_api::make_object(to_bytes(std::move(body_cell))); + data = tonlib_api::make_object(to_bytes(std::move(body_cell)), ""); } return data; }; @@ -2114,7 +2360,7 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactions& request, td::optional private_key; if (request.private_key_) { TRY_RESULT(input_key, from_tonlib(*request.private_key_)); - //NB: options has lot of problems. We use emplace to migitate them + //NB: optional has lot of problems. We use emplace to migitate them td::optional o_status; //NB: rely on (and assert) that GetPrivateKey is a synchonous request make_request(int_api::GetPrivateKey{std::move(input_key)}, [&](auto&& r_key) { @@ -2217,6 +2463,7 @@ class GenericCreateSendGrams : public TonlibQueryActor { std::string message; td::Ref body; + td::Ref init_state; td::optional o_public_key; }; @@ -2227,6 +2474,10 @@ class GenericCreateSendGrams : public TonlibQueryActor { // Should be splitted eventually std::vector dns_actions_; + bool pchan_action_{false}; + + bool rwallet_action_{false}; + void check(td::Status status) { if (status.is_error()) { promise_.set_error(std::move(status)); @@ -2257,34 +2508,36 @@ class GenericCreateSendGrams : public TonlibQueryActor { auto key = td::Ed25519::PublicKey(td::SecureString(public_key.key)); res.o_public_key = std::move(key); } - auto status = - downcast_call2(*message.data_, td::overloaded( - [&](tonlib_api::msg_dataRaw& text) { - TRY_RESULT(body, vm::std_boc_deserialize(text.body_)); - res.body = std::move(body); - return td::Status::OK(); - }, - [&](tonlib_api::msg_dataText& text) { - res.message = text.text_; - res.should_encrypt = false; - res.is_encrypted = false; - return td::Status::OK(); - }, - [&](tonlib_api::msg_dataDecryptedText& text) { - res.message = text.text_; - if (!has_private_key_) { - return TonlibError::EmptyField("input_key"); - } - res.should_encrypt = true; - res.is_encrypted = true; - return td::Status::OK(); - }, - [&](tonlib_api::msg_dataEncryptedText& text) { - res.message = text.text_; - res.should_encrypt = false; - res.is_encrypted = true; - return td::Status::OK(); - })); + auto status = downcast_call2( + *message.data_, td::overloaded( + [&](tonlib_api::msg_dataRaw& text) { + TRY_RESULT(body, vm::std_boc_deserialize(text.body_)); + TRY_RESULT(init_state, vm::std_boc_deserialize(text.init_state_, true)); + res.body = std::move(body); + res.init_state = std::move(init_state); + return td::Status::OK(); + }, + [&](tonlib_api::msg_dataText& text) { + res.message = text.text_; + res.should_encrypt = false; + res.is_encrypted = false; + return td::Status::OK(); + }, + [&](tonlib_api::msg_dataDecryptedText& text) { + res.message = text.text_; + if (!has_private_key_) { + return TonlibError::EmptyField("input_key"); + } + res.should_encrypt = true; + res.is_encrypted = true; + return td::Status::OK(); + }, + [&](tonlib_api::msg_dataEncryptedText& text) { + res.message = text.text_; + res.should_encrypt = false; + res.is_encrypted = true; + return td::Status::OK(); + })); // Use this limit as a preventive check if (res.message.size() > ton::Wallet::max_message_size) { return TonlibError::MessageTooLong(); @@ -2332,6 +2585,14 @@ class GenericCreateSendGrams : public TonlibQueryActor { } return td::Status::OK(); }, + [&](tonlib_api::actionPchan& cell) { + pchan_action_ = true; + return td::Status::OK(); + }, + [&](tonlib_api::actionRwallet& cell) { + rwallet_action_ = true; + return td::Status::OK(); + }, [&](tonlib_api::actionDns& cell) { for (auto& from_action : cell.actions_) { if (!from_action) { @@ -2397,6 +2658,9 @@ class GenericCreateSendGrams : public TonlibQueryActor { td::Status do_on_source_state(td::Result> r_state) { TRY_RESULT(state, std::move(r_state)); source_ = std::move(state); + if (source_->get_wallet_type() == AccountState::Empty && query_.initial_account_state_) { + source_->guess_type_by_init_state(*query_.initial_account_state_); + } if (source_->get_wallet_type() == AccountState::Empty && public_key_) { source_->guess_type_by_public_key(public_key_.value()); } @@ -2459,6 +2723,165 @@ class GenericCreateSendGrams : public TonlibQueryActor { return td::Status::OK(); } + td::Status do_pchan_loop(td::Ref pchan, tonlib_api::actionPchan& action) { + if (!action.action_) { + return TonlibError::EmptyField("action"); + } + + Query::Raw raw; + auto valid_until = source_->get_sync_time(); + valid_until += query_.timeout_ == 0 ? 60 : query_.timeout_; + raw.valid_until = valid_until; + + TRY_RESULT(info, pchan->get_info()); + bool is_alice = false; + bool is_bob = false; + if (info.config.a_key == private_key_.value().get_public_key().move_as_ok().as_octet_string()) { + LOG(ERROR) << "Alice key"; + is_alice = true; + } else if (info.config.b_key == private_key_.value().get_public_key().move_as_ok().as_octet_string()) { + LOG(ERROR) << "Bob key"; + is_bob = true; + } + if (!is_alice && !is_bob) { + return TonlibError::InvalidField("private_key", "invalid for this smartcontract"); + } + auto status = downcast_call2( + *action.action_, + td::overloaded( + [&](tonlib_api::pchan_actionTimeout& timeout) { + auto builder = ton::pchan::MsgTimeoutBuilder(); + if (is_alice) { + std::move(builder).with_a_key(&private_key_.value()); + } + if (is_bob) { + std::move(builder).with_b_key(&private_key_.value()); + } + raw.message_body = std::move(builder).finalize(); + return td::Status::OK(); + }, + [&](tonlib_api::pchan_actionInit& init) { + auto builder = ton::pchan::MsgInitBuilder() + .inc_A(init.inc_A_) + .inc_B(init.inc_B_) + .min_A(init.min_A_) + .min_B(init.min_B_) + .channel_id(info.config.channel_id); + if (is_alice) { + std::move(builder).with_a_key(&private_key_.value()); + } + if (is_bob) { + std::move(builder).with_b_key(&private_key_.value()); + } + raw.message_body = std::move(builder).finalize(); + return td::Status::OK(); + }, + [&](tonlib_api::pchan_actionClose& close) { + if (!close.promise_) { + return TonlibError::EmptyField("promise"); + } + + ton::pchan::SignedPromiseBuilder sbuilder; + sbuilder.promise_A(close.promise_->promise_A_) + .promise_B(close.promise_->promise_B_) + .channel_id(close.promise_->channel_id_) + .signature(td::SecureString(close.promise_->signature_)); + if (is_alice && !sbuilder.check_signature(close.promise_->signature_, + td::Ed25519::PublicKey(info.config.b_key.copy()))) { + return TonlibError::InvalidSignature(); + } + if (is_bob && !sbuilder.check_signature(close.promise_->signature_, + td::Ed25519::PublicKey(info.config.a_key.copy()))) { + return TonlibError::InvalidSignature(); + } + + auto builder = ton::pchan::MsgCloseBuilder() + .extra_A(close.extra_A_) + .extra_B(close.extra_B_) + .signed_promise(sbuilder.finalize()); + if (is_alice) { + std::move(builder).with_a_key(&private_key_.value()); + } + if (is_bob) { + std::move(builder).with_b_key(&private_key_.value()); + } + raw.message_body = std::move(builder).finalize(); + return td::Status::OK(); + })); + TRY_STATUS(std::move(status)); + + raw.new_state = source_->get_new_state(); + raw.message = ton::GenericAccount::create_ext_message(source_->get_address(), raw.new_state, raw.message_body); + raw.source = std::move(source_); + + promise_.set_value(td::make_unique(std::move(raw))); + stop(); + return td::Status::OK(); + } + td::Status do_pchan_loop() { + if (!private_key_) { + return TonlibError::EmptyField("private_key"); + } + + auto pchan = ton::PaymentChannel::create(source_->get_smc_state()); + + return downcast_call2( + *query_.action_, td::overloaded([&](tonlib_api::actionNoop& cell) { return td::Status::OK(); }, + [&](auto& cell) { return td::Status::Error(); }, + [&](tonlib_api::actionPchan& cell) { return do_pchan_loop(pchan, cell); })); + } + + td::Status do_rwallet_action(td::Ref rwallet, tonlib_api::actionRwallet& action) { + if (!action.action_) { + return TonlibError::EmptyField("action"); + } + auto& init = *action.action_; + if (!init.config_) { + return TonlibError::EmptyField("config"); + } + TRY_RESULT_PREFIX(start_at, td::narrow_cast_safe(init.config_->start_at_), + TonlibError::InvalidField("start_at", "not a uint32")); + ton::RestrictedWallet::Config config; + config.start_at = start_at; + for (auto& limit : init.config_->limits_) { + if (!limit) { + return TonlibError::EmptyField("limits"); + } + TRY_RESULT_PREFIX(seconds, td::narrow_cast_safe(limit->seconds_), + TonlibError::InvalidField("seconds", "not a int32")); + TRY_RESULT_PREFIX(value, td::narrow_cast_safe(limit->value_), + TonlibError::InvalidField("value", "not a uint64")); + config.limits.emplace_back(seconds, value); + } + Query::Raw raw; + auto valid_until = source_->get_sync_time(); + valid_until += query_.timeout_ == 0 ? 60 : query_.timeout_; + raw.valid_until = valid_until; + + TRY_RESULT_PREFIX(message_body, rwallet->get_init_message(private_key_.value(), valid_until, config), + TonlibError::Internal("Invalid rwalet init query")); + raw.message_body = std::move(message_body); + raw.new_state = source_->get_new_state(); + raw.message = ton::GenericAccount::create_ext_message(source_->get_address(), raw.new_state, raw.message_body); + raw.source = std::move(source_); + raw.destinations = std::move(destinations_); + promise_.set_value(td::make_unique(std::move(raw))); + stop(); + return td::Status::OK(); + } + + td::Status do_rwallet_action() { + if (!private_key_) { + return TonlibError::EmptyField("private_key"); + } + auto rwallet = ton::RestrictedWallet::create(source_->get_smc_state()); + LOG(ERROR) << rwallet->get_address().rserialize(true); + return downcast_call2( + *query_.action_, + td::overloaded([&](auto& cell) { return td::Status::Error("UNREACHABLE"); }, + [&](tonlib_api::actionRwallet& cell) { return do_rwallet_action(rwallet, cell); })); + } + td::Status do_loop() { if (!source_ || destinations_left_ != 0) { return td::Status::OK(); @@ -2470,6 +2893,12 @@ class GenericCreateSendGrams : public TonlibQueryActor { if (source_->get_wallet_type() == AccountState::ManualDns) { return do_dns_loop(); } + if (source_->get_wallet_type() == AccountState::PaymentChannel) { + return do_pchan_loop(); + } + if (rwallet_action_) { + return do_rwallet_action(); + } switch (source_->get_wallet_type()) { case AccountState::Empty: @@ -2507,6 +2936,7 @@ class GenericCreateSendGrams : public TonlibQueryActor { } if (action.body.not_null()) { gift.body = action.body; + gift.init_state = action.init_state; } else if (action.should_encrypt) { LOG(ERROR) << "TRY ENCRYPT"; if (!private_key_) { @@ -2581,7 +3011,7 @@ class GenericCreateSendGrams : public TonlibQueryActor { return with_wallet(*source_->get_wallet()); } -}; +}; // namespace tonlib td::int64 TonlibClient::register_query(td::unique_ptr query) { auto query_id = ++next_query_id_; @@ -2594,8 +3024,9 @@ td::Result> TonlibClient::get_que if (it == queries_.end()) { return TonlibError::InvalidQueryId(); } - return tonlib_api::make_object(id, it->second->get_valid_until(), - it->second->get_body_hash().as_slice().str()); + return tonlib_api::make_object( + id, it->second->get_valid_until(), it->second->get_body_hash().as_slice().str(), + to_bytes(it->second->get_message_body()), to_bytes(it->second->get_init_state())); } void TonlibClient::finish_create_query(td::Result> r_query, @@ -2938,6 +3369,8 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, stack.write().push(std::move(e)); } args.set_stack(std::move(stack)); + args.set_balance(it->second->get_balance()); + args.set_now(it->second->get_sync_time()); auto res = smc->run_get_method(std::move(args)); // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; @@ -3057,6 +3490,76 @@ td::Status TonlibClient::do_request(const tonlib_api::dns_resolve& request, return td::Status::OK(); } +td::Status TonlibClient::do_request(tonlib_api::pchan_signPromise& request, + td::Promise>&& promise) { + if (!request.promise_) { + return TonlibError::EmptyField("promise"); + } + if (!request.input_key_) { + return TonlibError::EmptyField("input_key"); + } + TRY_RESULT(input_key, from_tonlib(*request.input_key_)); + make_request(int_api::GetPrivateKey{std::move(input_key)}, + promise.wrap([promise = std::move(request.promise_)](auto key) mutable { + auto private_key = td::Ed25519::PrivateKey(std::move(key.private_key)); + promise->signature_ = ton::pchan::SignedPromiseBuilder() + .promise_A(promise->promise_A_) + .promise_B(promise->promise_B_) + .channel_id(promise->channel_id_) + .with_key(&private_key) + .calc_signature() + .as_slice() + .str(); + return std::move(promise); + })); + return td::Status::OK(); +} +td::Status TonlibClient::do_request(tonlib_api::pchan_validatePromise& request, + td::Promise>&& promise) { + if (!request.promise_) { + return TonlibError::EmptyField("promise"); + } + TRY_RESULT(key_bytes, get_public_key(request.public_key_)); + auto key = td::Ed25519::PublicKey(td::SecureString(key_bytes.key)); + bool is_ok = ton::pchan::SignedPromiseBuilder() + .promise_A(request.promise_->promise_A_) + .promise_B(request.promise_->promise_B_) + .channel_id(request.promise_->channel_id_) + .check_signature(request.promise_->signature_, key); + if (!is_ok) { + return TonlibError::InvalidSignature(); + } + promise.set_value(tonlib_api::make_object()); + return td::Status::OK(); +} +td::Status TonlibClient::do_request(tonlib_api::pchan_packPromise& request, + td::Promise>&& promise) { + if (!request.promise_) { + return TonlibError::EmptyField("promise"); + } + promise.set_value(tonlib_api::make_object( + td::SecureString(to_bytes(ton::pchan::SignedPromiseBuilder() + .promise_A(request.promise_->promise_A_) + .promise_B(request.promise_->promise_B_) + .channel_id(request.promise_->channel_id_) + .signature(td::SecureString(request.promise_->signature_)) + .finalize())))); + return td::Status::OK(); +} + +td::Status TonlibClient::do_request(tonlib_api::pchan_unpackPromise& request, + td::Promise>&& promise) { + TRY_RESULT_PREFIX(body, vm::std_boc_deserialize(request.data_), TonlibError::InvalidBagOfCells("data")); + ton::pchan::SignedPromise spromise; + if (!spromise.unpack(body)) { + return TonlibError::InvalidField("data", "Can't unpack as a promise"); + } + promise.set_value(tonlib_api::make_object( + spromise.o_signature.value().as_slice().str(), spromise.promise.promise_A, spromise.promise.promise_B, + spromise.promise.channel_id)); + return td::Status::OK(); +} + td::Status TonlibClient::do_request(tonlib_api::sync& request, td::Promise>&& promise) { // ton.blockIdExt workchain:int32 shard:int64 seqno:int32 root_hash:bytes file_hash:bytes = ton.BlockIdExt; @@ -3067,11 +3570,6 @@ td::Status TonlibClient::do_request(tonlib_api::sync& request, return td::Status::OK(); } -td::Result public_key_from_bytes(td::Slice bytes) { - TRY_RESULT_PREFIX(key_bytes, block::PublicKey::from_bytes(bytes), TonlibError::Internal()); - return key_bytes; -} - td::Status TonlibClient::do_request(const tonlib_api::createNewKey& request, td::Promise>&& promise) { TRY_RESULT_PREFIX( diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index eaec1fcf..a1f9d875 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -74,6 +74,7 @@ class TonlibClient : public td::actor::Actor { LastBlockState last_state; std::string last_state_key; td::uint32 wallet_id; + std::string rwallet_init_public_key; }; private: @@ -307,6 +308,15 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::dns_resolve& request, td::Promise>&& promise); + + td::Status do_request(tonlib_api::pchan_signPromise& request, + td::Promise>&& promise); + td::Status do_request(tonlib_api::pchan_validatePromise& request, td::Promise>&& promise); + + td::Status do_request(tonlib_api::pchan_packPromise& request, td::Promise>&& promise); + td::Status do_request(tonlib_api::pchan_unpackPromise& request, + td::Promise>&& promise); + void do_dns_request(std::string name, td::int32 category, td::int32 ttl, td::optional block_id, block::StdAddress address, td::Promise>&& promise); struct DnsFinishData { diff --git a/tonlib/tonlib/TonlibError.h b/tonlib/tonlib/TonlibError.h index a1d98fb3..c151849a 100644 --- a/tonlib/tonlib/TonlibError.h +++ b/tonlib/tonlib/TonlibError.h @@ -32,7 +32,7 @@ // INVALID_ACCOUNT_ADDRESS // INVALID_CONFIG // INVALID_PEM_KEY -// INVALID_REVISION +// INVALID_SIGNATURE // MESSAGE_TOO_LONG // EMPTY_FIELD // INVALID_FIELD @@ -84,6 +84,9 @@ struct TonlibError { static td::Status InvalidRevision() { return td::Status::Error(400, "INVALID_REVISION"); } + static td::Status InvalidSignature() { + return td::Status::Error(400, "INVALID_SIGNATURE"); + } static td::Status NeedConfig() { return td::Status::Error(400, "NeedConfig"); } diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 44096f12..b5d23c90 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -44,6 +44,7 @@ #include "tonlib/ExtClientLazy.h" #include "smc-envelope/ManualDns.h" +#include "smc-envelope/PaymentChannel.h" #include "auto/tl/tonlib_api.hpp" @@ -51,6 +52,8 @@ #include #include +using tonlib_api::make_object; + // GR$ struct Grams { td::uint64 nano; @@ -139,6 +142,17 @@ class TonlibCli : public td::actor::Actor { }; std::vector keys_; + struct Address { + tonlib_api::object_ptr address; + std::string public_key; + td::SecureString secret; + auto input_key(td::Slice password = "") const { + return !secret.empty() ? make_object( + make_object(public_key, secret.copy()), td::SecureString(password)) + : nullptr; + } + }; + std::map>> query_handlers_; td::actor::ActorOwn raw_client_; @@ -207,7 +221,6 @@ class TonlibCli : public td::actor::Actor { td::make_unique(td::actor::actor_shared())); } - using tonlib_api::make_object; auto config = !options_.config.empty() ? make_object(options_.config, options_.name, options_.use_callbacks_for_network, options_.ignore_cache) @@ -230,6 +243,7 @@ class TonlibCli : public td::actor::Actor { wallet_id_ = static_cast(r_ok.ok()->config_info_->default_wallet_id_); } } + load_channnels(); td::TerminalIO::out() << "Tonlib is inited\n"; if (options_.one_shot) { td::actor::send_closure(actor_id(this), &TonlibCli::parse_line, td::BufferSlice(options_.cmd)); @@ -261,6 +275,41 @@ class TonlibCli : public td::actor::Actor { std::_Exit(2); } } + void dns_help() { + td::TerminalIO::out() << "dns help\n"; + td::TerminalIO::out() << "dns resolve ( | root) \n"; + td::TerminalIO::out() << "dns cmd \n"; + //td::TerminalIO::out() << "dns cmdlist {\\n} end\n"; + td::TerminalIO::out() << "dns cmdfile \n"; + td::TerminalIO::out() << "\t = set | delete.name | delete.all\n"; + td::TerminalIO::out() << "\t = DELETED | EMPTY | TEXT: | NEXT: | SMC: | " + "ADNL:\n"; + } + + void pchan_help() { + td::TerminalIO::out() << "pchan help\n"; + td::TerminalIO::out() << "pchan create " + " []\n"; + td::TerminalIO::out() << "pchan list\n"; + td::TerminalIO::out() << "pchan delete \n"; + + td::TerminalIO::out() << "pchan getstate \n"; + td::TerminalIO::out() << "pchan promise make (A|B) \n"; + td::TerminalIO::out() << "pchan promise check (A|B) \n"; + td::TerminalIO::out() << "pchan promise pack \n"; + td::TerminalIO::out() << "pchan promise unpack \n"; + td::TerminalIO::out() << "pchan cmd \n"; + td::TerminalIO::out() << "\tfor simplicity we assume that alice_key is same as wallet key alice uses\n"; + td::TerminalIO::out() << "\t = init \n"; + td::TerminalIO::out() << "\t | close promise []\n"; + td::TerminalIO::out() << "\t | timeout \n"; + } + + void rwallet_help() { + td::TerminalIO::out() << "rwallet help\n"; + td::TerminalIO::out() << "rwallet address \n"; + td::TerminalIO::out() << "rwallet init [: ...]\n"; + } void parse_line(td::BufferSlice line) { if (is_closing_) { @@ -313,13 +362,10 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << "getstate \tget state of wallet with requested key\n"; td::TerminalIO::out() << "guessrevision \tsearch of existing accounts corresponding to the given key\n"; td::TerminalIO::out() << "getaddress \tget address of wallet with requested key\n"; - td::TerminalIO::out() << "dns resolve ( | root) \n"; - td::TerminalIO::out() << "dns cmd \n"; - //td::TerminalIO::out() << "dns cmdlist {\\n} end\n"; - td::TerminalIO::out() << "dns cmdfile \n"; - td::TerminalIO::out() << "\t = set | delete.name | delete.all\n"; - td::TerminalIO::out() << "\t = DELETED | EMPTY | TEXT: | NEXT: | SMC: | " - "ADNL:\n"; + + dns_help(); + pchan_help(); + rwallet_help(); td::TerminalIO::out() << "blockmode auto|manual\tWith auto mode, all queries will be executed with respect to the latest block. " @@ -419,6 +465,10 @@ class TonlibCli : public td::actor::Actor { import_key_raw(parser.read_word(), std::move(cmd_promise)); } else if (cmd == "dns") { run_dns_cmd(parser, std::move(cmd_promise)); + } else if (cmd == "pchan") { + run_pchan_cmd(parser, std::move(cmd_promise)); + } else if (cmd == "rwallet") { + run_rwallet_cmd(parser, std::move(cmd_promise)); } else if (cmd == "gethistory") { get_history(parser.read_word(), std::move(cmd_promise)); } else if (cmd == "guessrevision") { @@ -431,15 +481,436 @@ class TonlibCli : public td::actor::Actor { } } + void rwallet_address(td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE(promise, address, to_account_address(parser.read_word(), false)); + auto public_key = parser.read_word().str(); + TRY_RESULT_PROMISE( + promise, addr, + sync_send_query(make_object( + make_object(address.public_key, public_key, wallet_id_), 1))); + td::TerminalIO::out() << addr->account_address_ << "\n"; + promise.set_value(td::Unit()); + } + + void rwallet_init(td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE(promise, address, to_account_address(parser.read_word(), false)); + auto public_key = parser.read_word().str(); + auto initial_state = + make_object(address.public_key, public_key, wallet_id_); + TRY_RESULT_PROMISE( + promise, addr, + sync_send_query(make_object( + make_object(address.public_key, public_key, wallet_id_), 1))); + + TRY_RESULT_PROMISE(promise, start_at, td::to_integer_safe(parser.read_word())); + std::vector> limits; + while (true) { + auto word = parser.read_word(); + if (word.empty()) { + break; + } + auto column_at = word.find(':'); + TRY_RESULT_PROMISE(promise, value, parse_grams(word.substr(column_at + 1))); + TRY_RESULT_PROMISE(promise, seconds, td::to_integer_safe(word.substr(0, column_at))); + limits.emplace_back(seconds, value.nano); + } + auto config = make_object(); + config->start_at_ = start_at; + for (auto limit : limits) { + config->limits_.push_back(make_object(limit.first, limit.second)); + } + auto action = + make_object(make_object(std::move(config))); + send_query(make_object(address.input_key(), std::move(addr), 60, std::move(action), + std::move(initial_state)), + promise.send_closure(actor_id(this), &TonlibCli::transfer2, false)); + } + + void run_rwallet_cmd(td::ConstParser& parser, td::Promise promise) { + auto cmd = parser.read_word(); + if (cmd == "help") { + rwallet_help(); + return promise.set_value(td::Unit()); + } + if (cmd == "address") { + return rwallet_address(parser, std::move(promise)); + } + if (cmd == "init") { + return rwallet_init(parser, std::move(promise)); + } + + promise.set_error(td::Status::Error("Unknown command")); + } + + void run_pchan_cmd(td::ConstParser& parser, td::Promise promise) { + auto cmd = parser.read_word(); + if (cmd == "help") { + pchan_help(); + return promise.set_value(td::Unit()); + } + if (cmd == "create") { + return pchan_create(parser, std::move(promise)); + } + if (cmd == "list") { + return pchan_list(std::move(promise)); + } + if (cmd == "delete") { + return pchan_delete(parser, std::move(promise)); + } + if (cmd == "promise") { + return pchan_promise(parser, std::move(promise)); + } + if (cmd == "getstate") { + return pchan_getstate(parser, std::move(promise)); + } + if (cmd == "cmd") { + TRY_RESULT_PROMISE(promise, pchan_id, to_pchan_id(parser.read_word())); + auto subcmd = parser.read_word(); + if (subcmd == "init") { + return pchan_init(pchan_id, parser, std::move(promise)); + } + if (subcmd == "close") { + return pchan_close(pchan_id, parser, std::move(promise)); + } + if (subcmd == "timeout") { + return pchan_timeout(pchan_id, parser, std::move(promise)); + } + } + + promise.set_error(td::Status::Error("Unknown command")); + } + + struct Channel { + std::string alice_public_key; + std::string alice_address; + td::optional alice_id; + + std::string bob_public_key; + std::string bob_address; + td::optional bob_id; + td::int32 init_timeout{0}; + td::int32 close_timeout{0}; + td::int64 channel_id; + + std::string address; + + td::Status parse(td::ConstParser& parser, bool gen_channel_id = false) { + alice_public_key = parser.read_word().str(); + alice_address = parser.read_word().str(); + bob_public_key = parser.read_word().str(); + bob_address = parser.read_word().str(); + TRY_RESULT_ASSIGN(init_timeout, td::to_integer_safe(parser.read_word())); + TRY_RESULT_ASSIGN(close_timeout, td::to_integer_safe(parser.read_word())); + if (parser.status().is_error()) { + return parser.status().clone(); + } + + auto channel_id_str = parser.read_word(); + if (channel_id_str.empty()) { + if (gen_channel_id) { + channel_id = static_cast(td::Random::secure_uint64()); + } else { + return td::Status::Error("Empty channel id"); + } + } else { + TRY_RESULT_ASSIGN(channel_id, td::to_integer_safe(channel_id_str)); + } + return td::Status::OK(); + } + void store(td::StringBuilder& sb) { + sb << alice_public_key << " " << alice_address << " " << bob_public_key << " " << bob_address << " " + << init_timeout << " " << close_timeout << " " << channel_id; + } + + friend td::StringBuilder& operator<<(td::StringBuilder& sb, const Channel& channel) { + sb << "\n\t" << td::tag("a_key", channel.alice_public_key) << td::tag("a_addr", channel.alice_address); + sb << "\n\t" << td::tag("b_key", channel.bob_public_key) << td::tag("b_addr", channel.bob_address); + if (channel.alice_id) { + sb << "\n\t" << td::tag("alice_id", channel.alice_id.value()); + } + if (channel.bob_id) { + sb << "\n\t" << td::tag("b_id", channel.bob_id.value()); + } + sb << "\n\t" << td::tag("init timeout", channel.init_timeout) << td::tag("close timeout", channel.close_timeout); + sb << "\n\t" << td::tag("channel id", channel.channel_id); + sb << "\n\t" << td::tag("addr", channel.address); + return sb; + } + + auto to_address() { + return make_object(address); + } + auto to_init_state() { + return make_object(make_object( + alice_public_key, make_object(alice_address), bob_public_key, + make_object(bob_address), init_timeout, close_timeout, channel_id)); + } + }; + void store_channels() { + td::SecureString buf(10000); + td::StringBuilder sb(buf.as_mutable_slice()); + for (auto& it : channels_) { + it.second.store(sb); + sb << "\n"; + } + LOG_IF(FATAL, sb.is_error()) << "StringBuilder overflow"; + td::atomic_write_file(channel_db_path(), sb.as_cslice()); + } + + void load_channnels() { + auto r_db = td::read_file_secure(channel_db_path()); + if (r_db.is_error()) { + return; + } + auto db = r_db.move_as_ok(); + td::ConstParser parser(db.as_slice()); + while (true) { + auto line = td::trim(parser.read_till_nofail('\n')); + parser.skip_nofail('\n'); + if (line.empty()) { + break; + } + td::ConstParser line_parser(line); + do_pchan_create(line_parser, false).ensure(); + } + } + + std::map channels_; + td::int32 next_channel_id_{0}; + + td::Result to_pchan_id(td::Slice pchan_id_str) { + TRY_RESULT(pchan_id, td::to_integer_safe(pchan_id_str)); + auto it = channels_.find(pchan_id); + if (it == channels_.end()) { + return td::Status::Error("Unknown channle id"); + } + return pchan_id; + } + + td::Status do_pchan_create(td::ConstParser& parser, bool gen_channel_id) { + Channel channel; + TRY_STATUS(channel.parse(parser, gen_channel_id)); + TRY_RESULT(addr, sync_send_query(make_object(channel.to_init_state(), -1))); + channel.address = addr->account_address_; + + auto find_id = [&](td::Slice public_key, td::Slice address) -> td::optional { + auto r_addr = to_account_address(public_key); + if (r_addr.is_error()) { + return {}; + } + auto addr = r_addr.move_as_ok().address->account_address_; + if (address != addr) { + return {}; + } + for (td::int32 i = 0; i < static_cast(keys_.size()); i++) { + if (keys_[i].public_key == public_key) { + return i; + } + } + return {}; + }; + channel.alice_id = find_id(channel.alice_public_key, channel.alice_address); + channel.bob_id = find_id(channel.bob_public_key, channel.bob_address); + channels_[next_channel_id_++] = std::move(channel); + return td::Status::OK(); + } + void pchan_getstate(td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE_PREFIX(promise, pchan_id, to_pchan_id(parser.read_word()), "pchan_id"); + auto& chan = channels_[pchan_id]; + get_state(chan.address, std::move(promise)); + } + void pchan_promise(td::ConstParser& parser, td::Promise promise) { + auto cmd = parser.read_word(); + if (cmd == "make") { + return pchan_promise_make(parser, std::move(promise)); + } + if (cmd == "check") { + return pchan_promise_check(parser, std::move(promise)); + } + if (cmd == "pack") { + return pchan_promise_pack(parser, std::move(promise)); + } + if (cmd == "unpack") { + return pchan_promise_unpack(parser, std::move(promise)); + } + promise.set_error(td::Status::Error("Unknown command")); + } + + void pchan_promise_make2(tonlib_api::object_ptr ans, td::Promise promise) { + td::TerminalIO::out() << "Signature (base64url):" << td::base64url_encode(ans->signature_) << "\n"; + send_query(make_object(std::move(ans)), promise.wrap([](auto&& ans) { + td::TerminalIO::out() << "Promise (base64url): " << td::base64url_encode(ans->bytes_) << "\n"; + return td::Unit(); + })); + } + + void pchan_promise_make(td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE_PREFIX(promise, pchan_id, to_pchan_id(parser.read_word()), "pchan_id"); + td::Slice a_or_b = parser.read_word(); + bool is_a; + if (a_or_b == "A") { + is_a = true; + } else if (a_or_b == "B") { + is_a = false; + } else { + promise.set_error(td::Status::Error("(A|B) expected")); + return; + } + TRY_RESULT_PROMISE_PREFIX(promise, promise_A, parse_grams(parser.read_word()), "A"); + TRY_RESULT_PROMISE_PREFIX(promise, promise_B, parse_grams(parser.read_word()), "B"); + + auto& chan = channels_[pchan_id]; + Address addr; + if (is_a) { + TRY_RESULT_PROMISE_PREFIX_ASSIGN(promise, addr, to_account_address(chan.alice_public_key, true), + "Don't have Alice's key"); + } else { + TRY_RESULT_PROMISE_PREFIX_ASSIGN(promise, addr, to_account_address(chan.bob_public_key, true), + "Don't have Bob's key"); + } + + send_query(make_object( + addr.input_key(), + make_object("", promise_A.nano, promise_B.nano, chan.channel_id)), + promise.send_closure(actor_id(this), &TonlibCli::pchan_promise_make2)); + } + void pchan_promise_check(td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE_PREFIX(promise, pchan_id, to_pchan_id(parser.read_word()), "pchan_id"); + td::Slice a_or_b = parser.read_word(); + bool is_a; + if (a_or_b == "A") { + is_a = true; + } else if (a_or_b == "B") { + is_a = false; + } else { + promise.set_error(td::Status::Error("(A|B) expected")); + return; + } + TRY_RESULT_PROMISE_PREFIX(promise, promise_A, parse_grams(parser.read_word()), "promise_A"); + TRY_RESULT_PROMISE_PREFIX(promise, promise_B, parse_grams(parser.read_word()), "promise_B"); + + auto& chan = channels_[pchan_id]; + std::string public_key = is_a ? chan.alice_public_key : chan.bob_public_key; + + TRY_RESULT_PROMISE_PREFIX(promise, signature, td::base64url_decode(parser.read_word()), "signature"); + send_query(make_object( + public_key, make_object(std::move(signature), promise_A.nano, + promise_B.nano, chan.channel_id)), + promise.wrap([](auto&& ans) { + td::TerminalIO::out() << "signature is OK\n"; + return td::Unit(); + })); + } + void pchan_promise_pack(td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE_PREFIX(promise, channel_id, td::to_integer_safe(parser.read_word()), "pchan_id"); + TRY_RESULT_PROMISE_PREFIX(promise, promise_A, parse_grams(parser.read_word()), "promise_A"); + TRY_RESULT_PROMISE_PREFIX(promise, promise_B, parse_grams(parser.read_word()), "promise_B"); + TRY_RESULT_PROMISE_PREFIX(promise, signature, base64url_decode(parser.read_word()), "signature"); + send_query(make_object(make_object( + std::move(signature), promise_A.nano, promise_B.nano, channel_id)), + promise.wrap([](auto packed) { + td::TerminalIO::out() << "packed promise: " << base64url_encode(packed->bytes_.as_slice()) << "\n"; + return td::Unit(); + })); + } + void pchan_promise_unpack(td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE_PREFIX(promise, packed_promise, base64url_decode(parser.read_word()), "promise"); + send_query(make_object(td::SecureString(packed_promise)), + promise.wrap([](auto unpacked) { + td::TerminalIO::out() << "unpacked promise:\n" + << "promise_A: " << Grams{static_cast(unpacked->promise_A_)} << "\n" + << "promise_B: " << Grams{static_cast(unpacked->promise_B_)} << "\n" + << "channel_id: " << unpacked->channel_id_ << "\n" + << "signature: " << td::base64url_encode(unpacked->signature_) << "\n"; + return td::Unit(); + })); + } + + void pchan_create(td::ConstParser& parser, td::Promise promise) { + TRY_STATUS_PROMISE(promise, do_pchan_create(parser, true)); + td::TerminalIO::out() << "Channel #" << next_channel_id_ - 1 << channels_[next_channel_id_ - 1] << "\n"; + store_channels(); + promise.set_value(td::Unit()); + } + void pchan_list(td::Promise promise) { + for (auto& it : channels_) { + td::TerminalIO::out() << "Channel #" << it.first << it.second << "\n"; + } + promise.set_value(td::Unit()); + } + void pchan_delete(td::ConstParser& parser, td::Promise promise) { + promise.set_error(td::Status::Error("TODO")); + } + + void pchan_init_2(Address addr, td::int32 pchan_id, td::int64 value, + tonlib_api::object_ptr query, td::Promise promise) { + std::vector> messages; + messages.push_back( + make_object(channels_[pchan_id].to_address(), "", value, + make_object(query->body_, query->init_state_))); + auto action = make_object(std::move(messages), true); + send_query( + make_object(addr.input_key(), std::move(addr.address), 60, std::move(action), nullptr), + promise.send_closure(actor_id(this), &TonlibCli::transfer2, false)); + } + + void pchan_init(td::int32 pchan_id, td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE_PREFIX(promise, addr, to_account_address(parser.read_word(), true), "key_id"); + TRY_RESULT_PROMISE_PREFIX(promise, A, parse_grams(parser.read_word()), "A"); + TRY_RESULT_PROMISE_PREFIX(promise, B, parse_grams(parser.read_word()), "B"); + TRY_RESULT_PROMISE_PREFIX(promise, min_A, parse_grams(parser.read_word()), "min_A"); + TRY_RESULT_PROMISE_PREFIX(promise, min_B, parse_grams(parser.read_word()), "min_B"); + + auto action = make_object( + make_object(A.nano, B.nano, min_A.nano, min_B.nano)); + + auto value = A.nano + B.nano; + send_query(make_object(addr.input_key(), channels_[pchan_id].to_address(), 60, + std::move(action), channels_[pchan_id].to_init_state()), + promise.send_closure(actor_id(this), &TonlibCli::pchan_init_2, std::move(addr), pchan_id, value)); + return; + } + + void pchan_close2(td::int32 pchan_id, Address addr, tonlib_api::object_ptr pchan_promise, + td::Promise promise) { + auto action = make_object( + make_object(0, 0, std::move(pchan_promise))); + //send_query(make_object(addr.input_key(), channels_[pchan_id].to_address(), 60, + //std::move(action), channels_[pchan_id].to_init_state()), + //promise.send_closure(actor_id(this), &TonlibCli::pchan_init_2, std::move(addr), pchan_id, 1000000000)); + send_query(make_object(addr.input_key(), channels_[pchan_id].to_address(), 60, + std::move(action), channels_[pchan_id].to_init_state()), + promise.send_closure(actor_id(this), &TonlibCli::transfer2, false)); + } + + void pchan_close(td::int32 pchan_id, td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE_PREFIX(promise, addr, to_account_address(parser.read_word(), true), "key_id"); + TRY_RESULT_PROMISE_PREFIX(promise, packed_promise, base64url_decode(parser.read_word()), "promise"); + send_query(make_object(td::SecureString(packed_promise)), + promise.send_closure(actor_id(this), &TonlibCli::pchan_close2, pchan_id, std::move(addr))); + } + + void pchan_timeout(td::int32 pchan_id, td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE_PREFIX(promise, addr, to_account_address(parser.read_word(), true), "key_id"); + auto action = make_object(make_object()); + send_query(make_object(addr.input_key(), channels_[pchan_id].to_address(), 60, + std::move(action), channels_[pchan_id].to_init_state()), + promise.send_closure(actor_id(this), &TonlibCli::pchan_init_2, std::move(addr), pchan_id, 1000000000)); + } + void run_dns_cmd(td::ConstParser& parser, td::Promise promise) { auto cmd = parser.read_word(); + if (cmd == "help") { + dns_help(); + return promise.set_value(td::Unit()); + } if (cmd == "cmd" || cmd == "cmdlist" || cmd == "cmdfile") { return dns_cmd(cmd, parser, std::move(promise)); } if (cmd == "resolve") { return dns_resolve(parser, std::move(promise)); } - promise.set_error(td::Status::Error("Unknown cmd")); + promise.set_error(td::Status::Error("Unknown command")); } void do_dns_resolve(std::string name, td::int16 category, td::int32 ttl, @@ -452,7 +923,7 @@ class TonlibCli : public td::actor::Actor { if (resolved->entries_[0]->entry_->get_id() == tonlib_api::dns_entryDataNextResolver::ID && ttl != 0) { td::TerminalIO::out() << "Redirect resolver\n"; auto entry = tonlib_api::move_object_as(resolved->entries_[0]->entry_); - send_query(tonlib_api::make_object(std::move(entry->resolver_), name, category, ttl), + send_query(make_object(std::move(entry->resolver_), name, category, ttl), promise.send_closure(actor_id(this), &TonlibCli::do_dns_resolve, name, category, 0)); return; } @@ -475,9 +946,9 @@ class TonlibCli : public td::actor::Actor { TRY_RESULT_PROMISE(promise, category, td::to_integer_safe(category_str)); std::vector> entries; - entries.push_back(tonlib_api::make_object( - "", -1, tonlib_api::make_object(std::move(address.address)))); - do_dns_resolve(name.str(), category, 10, tonlib_api::make_object(std::move(entries)), + entries.push_back(make_object( + "", -1, make_object(std::move(address.address)))); + do_dns_resolve(name.str(), category, 10, make_object(std::move(entries)), std::move(promise)); } void dns_cmd(td::Slice cmd, td::ConstParser& parser, td::Promise promise) { @@ -495,13 +966,13 @@ class TonlibCli : public td::actor::Actor { std::vector> actions; for (auto& action : actions_ext) { if (action.name.empty()) { - actions.push_back(tonlib_api::make_object()); + actions.push_back(make_object()); td::TerminalIO::out() << "Delete all dns entries\n"; } else if (action.category == 0) { - actions.push_back(tonlib_api::make_object(action.name, 0)); + actions.push_back(make_object(action.name, 0)); td::TerminalIO::out() << "Delete all dns enties with name: " << action.name << "\n"; } else if (!action.data) { - actions.push_back(tonlib_api::make_object(action.name, action.category)); + actions.push_back(make_object(action.name, action.category)); td::TerminalIO::out() << "Delete all dns enties with name and category: " << action.name << ":" << action.category << "\n"; } else { @@ -516,33 +987,33 @@ class TonlibCli : public td::actor::Actor { TRY_STATUS_PROMISE(promise, std::move(error)); td::TerminalIO::out() << "Set dns entry: " << action.name << ":" << action.category << " " << sb.as_cslice() << "\n"; - actions.push_back(tonlib_api::make_object( - tonlib_api::make_object(action.name, action.category, std::move(data)))); + actions.push_back(make_object( + make_object(action.name, action.category, std::move(data)))); } } - auto action = tonlib_api::make_object(std::move(actions)); + auto action = make_object(std::move(actions)); td::Slice password; // empty by default - using tonlib_api::make_object; + auto key = !address.secret.empty() ? make_object( make_object(address.public_key, address.secret.copy()), td::SecureString(password)) : nullptr; - send_query(tonlib_api::make_object(std::move(key), std::move(address.address), 60, - std::move(action)), + send_query(make_object(std::move(key), std::move(address.address), 60, std::move(action), + nullptr), promise.send_closure(actor_id(this), &TonlibCli::transfer2, false)); } void remote_time(td::Promise promise) { - send_query(tonlib_api::make_object(), promise.wrap([](auto&& info) { + send_query(make_object(), promise.wrap([](auto&& info) { td::TerminalIO::out() << "Lite server time is: " << info->now_ << "\n"; return td::Unit(); })); } void remote_version(td::Promise promise) { - send_query(tonlib_api::make_object(), promise.wrap([](auto&& info) { + send_query(make_object(), promise.wrap([](auto&& info) { td::TerminalIO::out() << "Lite server time is: " << info->now_ << "\n"; td::TerminalIO::out() << "Lite server version is: " << info->version_ << "\n"; td::TerminalIO::out() << "Lite server capabilities are: " << info->capabilities_ << "\n"; @@ -552,7 +1023,7 @@ class TonlibCli : public td::actor::Actor { void send_file(td::Slice name, td::Promise promise) { TRY_RESULT_PROMISE(promise, data, td::read_file_str(name.str())); - send_query(tonlib_api::make_object(std::move(data)), promise.wrap([](auto&& info) { + send_query(make_object(std::move(data)), promise.wrap([](auto&& info) { td::TerminalIO::out() << "Query was sent\n"; return td::Unit(); })); @@ -560,7 +1031,7 @@ class TonlibCli : public td::actor::Actor { void save_account(td::Slice cmd, td::Slice path, td::Slice address, td::Promise promise) { TRY_RESULT_PROMISE(promise, addr, to_account_address(address, false)); - send_query(tonlib_api::make_object(std::move(addr.address)), + send_query(make_object(std::move(addr.address)), promise.send_closure(actor_id(this), &TonlibCli::save_account_2, cmd.str(), path.str(), address.str())); } @@ -571,14 +1042,11 @@ class TonlibCli : public td::actor::Actor { promise.send_closure(actor_id(self), &TonlibCli::save_account_3, std::move(path), std::move(log))); }; if (cmd == "saveaccount") { - with_query(tonlib_api::make_object(info->id_), - PSTRING() << "StateInit of account " << address); + with_query(make_object(info->id_), PSTRING() << "StateInit of account " << address); } else if (cmd == "saveaccountcode") { - with_query(tonlib_api::make_object(info->id_), PSTRING() - << "Code of account " << address); + with_query(make_object(info->id_), PSTRING() << "Code of account " << address); } else if (cmd == "saveaccountdata") { - with_query(tonlib_api::make_object(info->id_), PSTRING() - << "Data of account " << address); + with_query(make_object(info->id_), PSTRING() << "Data of account " << address); } else { promise.set_error(td::Status::Error("Unknown query")); } @@ -593,7 +1061,6 @@ class TonlibCli : public td::actor::Actor { } void sync(td::Promise promise, bool update_last) { - using tonlib_api::make_object; send_query(make_object(), promise.wrap([&, update_last](auto&& block) { td::TerminalIO::out() << "synchronized\n"; td::TerminalIO::out() << to_string(block) << "\n"; @@ -625,8 +1092,8 @@ class TonlibCli : public td::actor::Actor { if (l == 1 || str.back() != '"' || l >= 127 + 2 || !cb.store_bytes_bool(str.data() + 1, l - 2)) { return td::Status::Error("Failed to parse slice"); } - return tonlib_api::make_object( - tonlib_api::make_object(vm::std_boc_serialize(cb.finalize()).ok().as_slice().str())); + return make_object( + make_object(vm::std_boc_serialize(cb.finalize()).ok().as_slice().str())); } if (l >= 3 && (str[0] == 'x' || str[0] == 'b') && str[1] == '{' && str.back() == '}') { unsigned char buff[128]; @@ -637,7 +1104,7 @@ class TonlibCli : public td::actor::Actor { if (bits < 0) { return td::Status::Error("Failed to parse slice"); } - return tonlib_api::make_object(tonlib_api::make_object( + return make_object(make_object( vm::std_boc_serialize(vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize()) .ok() .as_slice() @@ -657,8 +1124,7 @@ class TonlibCli : public td::actor::Actor { } else if (!l || x.parse_dec(str.data(), l) != l) { return td::Status::Error("Failed to parse a number"); } - return tonlib_api::make_object( - tonlib_api::make_object(dec_string(num))); + return make_object(make_object(dec_string(num))); } td::Result>> parse_stack(td::ConstParser& parser, @@ -672,12 +1138,12 @@ class TonlibCli : public td::actor::Actor { } if (word == "[") { TRY_RESULT(elements, parse_stack(parser, "]")); - stack.push_back(tonlib_api::make_object( - tonlib_api::make_object(std::move(elements)))); + stack.push_back( + make_object(make_object(std::move(elements)))); } else if (word == "(") { TRY_RESULT(elements, parse_stack(parser, ")")); - stack.push_back(tonlib_api::make_object( - tonlib_api::make_object(std::move(elements)))); + stack.push_back( + make_object(make_object(std::move(elements)))); } else { TRY_RESULT(stack_entry, parse_stack_entry(word)); stack.push_back(std::move(stack_entry)); @@ -734,9 +1200,9 @@ class TonlibCli : public td::actor::Actor { auto method_str = parser.read_word(); tonlib_api::object_ptr method; if (std::all_of(method_str.begin(), method_str.end(), [](auto c) { return c >= '0' && c <= '9'; })) { - method = tonlib_api::make_object(td::to_integer(method_str.str())); + method = make_object(td::to_integer(method_str.str())); } else { - method = tonlib_api::make_object(method_str.str()); + method = make_object(method_str.str()); } TRY_RESULT_PROMISE(promise, stack, parse_stack(parser, "")); td::StringBuilder sb; @@ -747,10 +1213,9 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << "Run " << to_string(method) << "With stack:\n" << sb.as_cslice(); - auto to_run = - tonlib_api::make_object(0 /*fixme*/, std::move(method), std::move(stack)); + auto to_run = make_object(0 /*fixme*/, std::move(method), std::move(stack)); - send_query(tonlib_api::make_object(std::move(addr.address)), + send_query(make_object(std::move(addr.address)), promise.send_closure(actor_id(this), &TonlibCli::run_method_2, std::move(to_run))); } @@ -775,7 +1240,6 @@ class TonlibCli : public td::actor::Actor { void set_validate_config(td::Slice cmd, td::Slice path, td::Slice name, bool use_callback, bool ignore_cache, td::Promise promise) { TRY_RESULT_PROMISE(promise, data, td::read_file_str(path.str())); - using tonlib_api::make_object; auto config = make_object(std::move(data), name.str(), use_callback, ignore_cache); if (cmd == "setconfig") { @@ -796,7 +1260,6 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << td::tag("rcv", td::format::as_size(rcv_bytes_)) << "\n"; } void on_adnl_result(td::uint64 id, td::Result res) { - using tonlib_api::make_object; if (res.is_ok()) { rcv_bytes_ += res.ok().size(); send_query(make_object(id, res.move_as_ok().as_slice().str()), @@ -884,10 +1347,10 @@ class TonlibCli : public td::actor::Actor { promise.set_error(td::Status::Error("empty current block")); return; } - func = tonlib_api::make_object( - tonlib_api::make_object(current_block_->workchain_, current_block_->shard_, - current_block_->seqno_, current_block_->root_hash_, - current_block_->file_hash_), + func = make_object( + make_object(current_block_->workchain_, current_block_->shard_, + current_block_->seqno_, current_block_->root_hash_, + current_block_->file_hash_), std::move(func)); } auto query_id = next_query_id_++; @@ -901,8 +1364,26 @@ class TonlibCli : public td::actor::Actor { }; } + template + td::Result sync_send_query(tonlib_api::object_ptr query) { + if (is_closing_) { + return td::Status::Error("Closing"); + } + auto r_obj = tonlib::TonlibClient::static_request(std::move(query)); + if (r_obj->get_id() == tonlib_api::error::ID) { + auto err = ton::move_tl_object_as(std::move(r_obj)); + return td::Status::Error(err->code_, err->message_); + } + return ton::move_tl_object_as(r_obj); + } + + td::Status validate_address(td::Slice addr) { + TRY_STATUS(sync_send_query(make_object(addr.str()))); + return td::Status::OK(); + } + void unpack_address(td::Slice addr) { - send_query(tonlib_api::make_object(addr.str()), + send_query(make_object(addr.str()), [addr = addr.str()](auto r_parsed_addr) mutable { if (r_parsed_addr.is_error()) { LOG(ERROR) << "Failed to parse address: " << r_parsed_addr.error(); @@ -913,23 +1394,22 @@ class TonlibCli : public td::actor::Actor { } void set_bounceable(td::Slice addr, bool bounceable) { - send_query(tonlib_api::make_object(addr.str()), - [addr = addr.str(), bounceable, this](auto r_parsed_addr) mutable { - if (r_parsed_addr.is_error()) { - LOG(ERROR) << "Failed to parse address: " << r_parsed_addr.error(); - return; - } - auto parsed_addr = r_parsed_addr.move_as_ok(); - parsed_addr->bounceable_ = bounceable; - this->send_query(tonlib_api::make_object(std::move(parsed_addr)), - [](auto r_addr) mutable { - if (r_addr.is_error()) { - LOG(ERROR) << "Failed to pack address"; - return; - } - td::TerminalIO::out() << r_addr.ok()->account_address_ << "\n"; - }); - }); + send_query(make_object(addr.str()), [addr = addr.str(), bounceable, + this](auto r_parsed_addr) mutable { + if (r_parsed_addr.is_error()) { + LOG(ERROR) << "Failed to parse address: " << r_parsed_addr.error(); + return; + } + auto parsed_addr = r_parsed_addr.move_as_ok(); + parsed_addr->bounceable_ = bounceable; + this->send_query(make_object(std::move(parsed_addr)), [](auto r_addr) mutable { + if (r_addr.is_error()) { + LOG(ERROR) << "Failed to pack address"; + return; + } + td::TerminalIO::out() << r_addr.ok()->account_address_ << "\n"; + }); + }); } void generate_key(td::SecureString entropy = {}) { @@ -951,8 +1431,8 @@ class TonlibCli : public td::actor::Actor { void generate_key(td::SecureString entropy, td::SecureString password) { auto password_copy = password.copy(); - send_query(tonlib_api::make_object( - std::move(password_copy), td::SecureString() /*mnemonic password*/, std::move(entropy)), + send_query(make_object(std::move(password_copy), td::SecureString() /*mnemonic password*/, + std::move(entropy)), [this, password = std::move(password)](auto r_key) mutable { if (r_key.is_error()) { LOG(ERROR) << "Failed to create new key: " << r_key.error(); @@ -1037,7 +1517,7 @@ class TonlibCli : public td::actor::Actor { } void do_delete_all_keys() { - send_query(tonlib_api::make_object(), [](auto r_res) { + send_query(make_object(), [](auto r_res) { if (r_res.is_error()) { td::TerminalIO::out() << "Something went wrong: " << r_res.error() << "\n"; return; @@ -1049,6 +1529,9 @@ class TonlibCli : public td::actor::Actor { std::string key_db_path() { return options_.key_dir + TD_DIR_SLASH + "key_db"; } + std::string channel_db_path() { + return options_.key_dir + TD_DIR_SLASH + "channel_db"; + } td::Result to_key_i(td::Slice key) { if (key.empty()) { @@ -1088,15 +1571,8 @@ class TonlibCli : public td::actor::Actor { return res; } - struct Address { - tonlib_api::object_ptr address; - std::string public_key; - td::SecureString secret; - }; - template auto with_account_state(int version, std::string public_key, td::uint32 wallet_id, F&& f) { - using tonlib_api::make_object; if (version == 1) { return f(make_object(public_key)); } @@ -1115,6 +1591,20 @@ class TonlibCli : public td::actor::Actor { return f(make_object(public_key, wallet_id)); } + td::Result
to_account_address(td::Slice public_key) { + auto r_addr = [&, self = this](td::int32 version, td::int32 revision) { + auto do_request = [revision, self](auto x) { + return self->sync_send_query(make_object(std::move(x), revision)); + }; + return with_account_state(version, public_key.str(), wallet_id_, do_request); + }(options_.wallet_version, options_.wallet_revision); + TRY_RESULT(addr, std::move(r_addr)); + Address res; + res.address = std::move(addr); + res.public_key = public_key.str(); + return std::move(res); + } + td::Result
to_account_address(td::Slice key, bool need_private_key) { if (key.empty()) { return td::Status::Error("account address is empty"); @@ -1122,20 +1612,29 @@ class TonlibCli : public td::actor::Actor { if (key == "none" && !need_private_key) { return Address{}; } + + auto at_pos = key.find('@'); + td::optional address; + if (at_pos != td::Slice::npos) { + address = key.substr(at_pos + 1).str(); + key.truncate(at_pos); + } + auto r_key_i = to_key_i(key); - using tonlib_api::make_object; + if (r_key_i.is_ok()) { - auto obj = [&](td::int32 version, td::int32 revision) { - auto do_request = [revision](auto x) { - return tonlib::TonlibClient::static_request( - make_object(std::move(x), revision)); - }; - return with_account_state(version, keys_[r_key_i.ok()].public_key, wallet_id_, do_request); - }(options_.wallet_version, options_.wallet_revision); - if (obj->get_id() != tonlib_api::error::ID) { + auto& key = keys_[r_key_i.ok()]; + if (address) { Address res; - res.address = ton::move_tl_object_as(obj); - res.public_key = keys_[r_key_i.ok()].public_key; + res.public_key = key.public_key; + res.secret = key.secret.copy(); + res.address = make_object(address.unwrap()); + return std::move(res); + } + + auto r_addr = to_account_address(key.public_key); + if (r_addr.is_ok()) { + Address res = r_addr.move_as_ok(); res.secret = keys_[r_key_i.ok()].secret.copy(); return std::move(res); } @@ -1151,6 +1650,12 @@ class TonlibCli : public td::actor::Actor { LOG(ERROR) << "Unexpected error during testGiver_getAccountAddress: " << to_string(obj); } } + if (!need_private_key) { + auto r_addr = to_account_address(key); + if (r_addr.is_ok()) { + return r_addr.move_as_ok(); + } + } if (need_private_key) { return td::Status::Error("Don't have a private key for this address"); } @@ -1166,7 +1671,7 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << "Unknown key id: [" << key << "]\n"; return; } - using tonlib_api::make_object; + auto key_i = r_key_i.move_as_ok(); send_query(make_object( make_object(keys_[key_i].public_key, keys_[key_i].secret.copy())), @@ -1202,7 +1707,7 @@ class TonlibCli : public td::actor::Actor { void import_key_pem(td::Slice filename, td::Promise promise) { TRY_RESULT_PROMISE(promise, data, td::read_file_secure(filename.str())); - using tonlib_api::make_object; + send_query(make_object(td::SecureString(), td::SecureString("cucumber"), make_object(std::move(data))), promise.wrap([&](auto&& key) { @@ -1218,7 +1723,7 @@ class TonlibCli : public td::actor::Actor { } void import_key_raw(td::Slice filename, td::Promise promise) { TRY_RESULT_PROMISE(promise, data, td::read_file_secure(filename.str())); - using tonlib_api::make_object; + send_query(make_object( td::SecureString(), make_object(std::move(data))), promise.wrap([&](auto&& key) { @@ -1234,7 +1739,6 @@ class TonlibCli : public td::actor::Actor { } void export_key(std::string cmd, std::string key, size_t key_i, td::Slice password) { - using tonlib_api::make_object; if (cmd == "exportkey") { send_query(make_object(make_object( make_object(keys_[key_i].public_key, keys_[key_i].secret.copy()), @@ -1287,7 +1791,6 @@ class TonlibCli : public td::actor::Actor { } void import_key(std::vector words, td::Slice password) { - using tonlib_api::make_object; send_query(make_object(td::SecureString(password), td::SecureString(""), make_object(std::move(words))), [this, password = td::SecureString(password)](auto r_res) { @@ -1308,7 +1811,7 @@ class TonlibCli : public td::actor::Actor { void get_state(td::Slice key, td::Promise promise) { TRY_RESULT_PROMISE(promise, address, to_account_address(key, false)); - using tonlib_api::make_object; + auto address_str = address.address->account_address_; send_query(make_object( ton::move_tl_object_as(std::move(address.address))), @@ -1334,7 +1837,7 @@ class TonlibCli : public td::actor::Actor { void get_history(td::Slice key, td::Promise promise) { TRY_RESULT_PROMISE(promise, address, to_account_address(key, false)); - using tonlib_api::make_object; + send_query(make_object( ton::move_tl_object_as(std::move(address.address))), promise.send_closure(td::actor::actor_id(this), &TonlibCli::get_history2, key.str())); @@ -1343,7 +1846,6 @@ class TonlibCli : public td::actor::Actor { void guess_revision(td::Slice key, td::Promise promise) { TRY_RESULT_PROMISE(promise, key_i, to_key_i(key)); with_account_state(options_.wallet_version, keys_[key_i].public_key, wallet_id_, [&](auto state) { - using tonlib_api::make_object; send_query(make_object(std::move(state)), promise.wrap([](auto revisions) { td::TerminalIO::out() << to_string(revisions); return td::Unit(); @@ -1360,14 +1862,10 @@ class TonlibCli : public td::actor::Actor { } TRY_RESULT_PROMISE(promise, address, std::move(r_address)); td::Slice password; - using tonlib_api::make_object; - auto input_key = - !address.secret.empty() - ? make_object( - make_object(address.public_key, address.secret.copy()), td::SecureString(password)) - : nullptr; - send_query(tonlib_api::make_object( + auto input_key = address.input_key(password); + + send_query(make_object( std::move(input_key), ton::move_tl_object_as(std::move(address.address)), std::move(state->last_transaction_id_)), promise.wrap([](auto res) { @@ -1384,7 +1882,7 @@ class TonlibCli : public td::actor::Actor { sb << "-" << Grams{td::uint64(-balance)}; } sb << " Fee: " << Grams{td::uint64(t->fee_)}; - if (!t->in_msg_->source_->account_address_.empty()) { + if (t->in_msg_->source_->account_address_.empty()) { sb << " External "; } else { sb << " From " << t->in_msg_->source_->account_address_; @@ -1472,12 +1970,12 @@ class TonlibCli : public td::actor::Actor { tonlib_api::object_ptr data; if (use_encryption) { - data = tonlib_api::make_object(message.str()); + data = make_object(message.str()); } else { - data = tonlib_api::make_object(message.str()); + data = make_object(message.str()); } - messages.push_back(tonlib_api::make_object(std::move(address.address), "", amount.nano, - std::move(data))); + messages.push_back( + make_object(std::move(address.address), "", amount.nano, std::move(data))); return td::Status::OK(); }; @@ -1502,7 +2000,7 @@ class TonlibCli : public td::actor::Actor { } td::Slice password; // empty by default - using tonlib_api::make_object; + tonlib_api::object_ptr key = !from_address.secret.empty() ? make_object( @@ -1517,20 +2015,19 @@ class TonlibCli : public td::actor::Actor { send_query(make_object( std::move(key), std::move(from_address.address), 60, - make_object(std::move(messages), allow_send_to_uninited)), + make_object(std::move(messages), allow_send_to_uninited), nullptr), cmd_promise.send_closure(actor_id(this), &TonlibCli::transfer2, estimate_fees)); } void transfer2(bool estimate_fees, td::Result> r_info, td::Promise cmd_promise) { if (estimate_fees) { - send_query(tonlib_api::make_object(r_info.ok()->id_, true), - cmd_promise.wrap([](auto&& info) { - td::TerminalIO::out() << "Extimated fees: " << to_string(info); - return td::Unit(); - })); + send_query(make_object(r_info.ok()->id_, true), cmd_promise.wrap([](auto&& info) { + td::TerminalIO::out() << "Extimated fees: " << to_string(info); + return td::Unit(); + })); } else { - send_query(tonlib_api::make_object(r_info.ok()->id_), cmd_promise.wrap([](auto&& info) { + send_query(make_object(r_info.ok()->id_), cmd_promise.wrap([](auto&& info) { td::TerminalIO::out() << "Transfer sent: " << to_string(info); return td::Unit(); })); @@ -1538,7 +2035,6 @@ class TonlibCli : public td::actor::Actor { } void get_hints(td::Slice prefix) { - using tonlib_api::make_object; auto obj = tonlib::TonlibClient::static_request(make_object(prefix.str())); if (obj->get_id() == tonlib_api::error::ID) { return; @@ -1548,7 +2044,6 @@ class TonlibCli : public td::actor::Actor { }; int main(int argc, char* argv[]) { - using tonlib_api::make_object; SET_VERBOSITY_LEVEL(verbosity_INFO); td::set_default_failure_signal_handler(); diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index e02c0005..3fd1c45d 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1172,13 +1172,16 @@ td::Status ValidatorEngine::load_global_config() { validator_options_ = ton::validator::ValidatorManagerOptions::create(zero_state, init_block); validator_options_.write().set_shard_check_function( - [](ton::ShardIdFull shard, ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { + [](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, + ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { return true; } CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); - //return shard.is_masterchain(); return true; + /*ton::ShardIdFull p{ton::basechainId, ((cc_seqno * 1ull % 4) << 62) + 1}; + auto s = ton::shard_prefix(p, 2); + return shard.is_masterchain() || ton::shard_intersects(shard, s);*/ }); if (state_ttl_ != 0) { validator_options_.write().set_state_ttl(state_ttl_); @@ -1195,12 +1198,12 @@ td::Status ValidatorEngine::load_global_config() { if (key_proof_ttl_ != 0) { validator_options_.write().set_key_proof_ttl(key_proof_ttl_); } - if (db_depth_ <= 32) { - validator_options_.write().set_filedb_depth(db_depth_); - } for (auto seq : unsafe_catchains_) { validator_options_.write().add_unsafe_resync_catchain(seq); } + if (truncate_seqno_ > 0) { + validator_options_.write().truncate_db(truncate_seqno_); + } std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -3079,6 +3082,10 @@ std::atomic rotate_logs_flags{false}; void force_rotate_logs(int sig) { rotate_logs_flags.store(true); } +std::atomic need_scheduler_status_flag{false}; +void need_scheduler_status(int sig) { + need_scheduler_status_flag.store(true); +} void dump_memory_stats() { if (!is_memprof_on()) { @@ -3164,15 +3171,6 @@ int main(int argc, char *argv[]) { acts.push_back([&x, fname = fname.str()]() { td::actor::send_closure(x, &ValidatorEngine::set_fift_dir, fname); }); return td::Status::OK(); }); - p.add_option('F', "filedb-depth", - "depth of autodirs for blocks, proofs, etc. Default value is 2. You need to clear the " - "database, if you need to change this option", - [&](td::Slice fname) { - acts.push_back([&x, fname = fname.str()]() { - td::actor::send_closure(x, &ValidatorEngine::set_db_depth, td::to_integer(fname)); - }); - return td::Status::OK(); - }); p.add_option('d', "daemonize", "set SIGHUP", [&]() { #if TD_DARWIN || TD_LINUX close(0); @@ -3215,6 +3213,12 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_ttl, v); }); return td::Status::OK(); }); + p.add_option('T', "truncate-db", "truncate db (with specified seqno as new top masterchain block seqno)", + [&](td::Slice fname) { + auto v = td::to_integer(fname); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_truncate_seqno, v); }); + return td::Status::OK(); + }); p.add_option('U', "unsafe-catchain-restore", "use SLOW and DANGEROUS catchain recover method", [&](td::Slice id) { TRY_RESULT(seq, td::to_integer_safe(id)); acts.push_back([&x, seq]() { td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain, seq); }); @@ -3242,7 +3246,9 @@ int main(int argc, char *argv[]) { } td::set_runtime_signal_handler(1, need_stats).ensure(); + td::set_runtime_signal_handler(2, need_scheduler_status).ensure(); + td::actor::set_debug(true); td::actor::Scheduler scheduler({threads}); scheduler.run_in_context([&] { @@ -3258,6 +3264,10 @@ int main(int argc, char *argv[]) { if (need_stats_flag.exchange(false)) { dump_stats(); } + if (need_scheduler_status_flag.exchange(false)) { + LOG(ERROR) << "DUMPING SCHEDULER STATISTICS"; + scheduler.get_debug().dump(); + } if (rotate_logs_flags.exchange(false)) { if (td::log_interface) { td::log_interface->rotate(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index eac7b499..7cda92b9 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -195,10 +195,10 @@ class ValidatorEngine : public td::actor::Actor { td::Clocks::Duration sync_ttl_ = 0; td::Clocks::Duration archive_ttl_ = 0; td::Clocks::Duration key_proof_ttl_ = 0; - td::uint32 db_depth_ = 33; bool read_config_ = false; bool started_keyring_ = false; bool started_ = false; + ton::BlockSeqno truncate_seqno_{0}; std::set unsafe_catchains_; @@ -216,9 +216,6 @@ class ValidatorEngine : public td::actor::Actor { fift_dir_ = str; } void set_db_root(std::string db_root); - void set_db_depth(td::uint32 value) { - db_depth_ = value; - } void set_state_ttl(td::Clocks::Duration t) { state_ttl_ = t; } @@ -234,6 +231,9 @@ class ValidatorEngine : public td::actor::Actor { void set_key_proof_ttl(td::Clocks::Duration t) { key_proof_ttl_ = t; } + void set_truncate_seqno(ton::BlockSeqno seqno) { + truncate_seqno_ = seqno; + } void add_ip(td::IPAddress addr) { addrs_.push_back(addr); } diff --git a/validator-session/validator-session-description.cpp b/validator-session/validator-session-description.cpp index c1a45dd6..42059053 100644 --- a/validator-session/validator-session-description.cpp +++ b/validator-session/validator-session-description.cpp @@ -134,7 +134,7 @@ std::vector ValidatorSessionDescriptionImpl::export_full_nodes() cons } double ValidatorSessionDescriptionImpl::get_delay(td::uint32 priority) const { - return priority * opts_.next_candidate_delay; + return ((sources_.size() >= 5 ? 0 : 1) + priority) * opts_.next_candidate_delay; } td::uint32 ValidatorSessionDescriptionImpl::get_vote_for_author(td::uint32 attempt_seqno) const { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index a1bd1fe3..65d7b348 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -110,6 +110,25 @@ set(DISK_VALIDATOR_SOURCE ${VALIDATOR_DB_SOURCE} ) +set(HARDFORK_VALIDATOR_SOURCE + apply-block.cpp + block-handle.cpp + shard-client.cpp + validator-full-id.cpp + validator-group.cpp + validator-options.cpp + + downloaders/wait-block-data-disk.cpp + downloaders/wait-block-state.cpp + downloaders/wait-block-state-merge.cpp + downloaders/download-state.cpp + + manager-init.cpp + manager-hardfork.cpp + + ${VALIDATOR_DB_SOURCE} +) + set(FULL_NODE_SOURCE full-node.h full-node.hpp @@ -139,6 +158,7 @@ set(FULL_NODE_SOURCE add_library(validator STATIC ${VALIDATOR_SOURCE}) add_library(validator-disk STATIC ${DISK_VALIDATOR_SOURCE}) +add_library(validator-hardfork STATIC ${HARDFORK_VALIDATOR_SOURCE}) add_library(full-node STATIC ${FULL_NODE_SOURCE}) target_include_directories(validator PUBLIC @@ -155,6 +175,13 @@ target_include_directories(validator-disk PUBLIC ${OPENSSL_INCLUDE_DIR} ) +target_include_directories(validator-hardfork PUBLIC + $ + $/.. + $/../crypto + ${OPENSSL_INCLUDE_DIR} + ) + target_include_directories(full-node PUBLIC $ $/.. @@ -168,5 +195,8 @@ target_link_libraries(validator PRIVATE tdutils tdactor adnl rldp tl_api dht tdf target_link_libraries(validator-disk PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_crypto ton_block ton_db) +target_link_libraries(validator-hardfork PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec + overlay catchain validatorsession ton_crypto ton_block ton_db) + target_link_libraries(full-node PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_crypto ton_block ton_db) diff --git a/validator/block-handle.cpp b/validator/block-handle.cpp index f7749e0c..7fc770cc 100644 --- a/validator/block-handle.cpp +++ b/validator/block-handle.cpp @@ -42,8 +42,8 @@ td::BufferSlice BlockHandleImpl::serialize() const { (flags & dbf_inited_masterchain_ref_block) ? masterchain_ref_seqno_ : 0); } -BlockHandleImpl::BlockHandleImpl(td::BufferSlice data) { - auto obj = fetch_tl_object(std::move(data), true).move_as_ok(); +BlockHandleImpl::BlockHandleImpl(td::Slice data) { + auto obj = fetch_tl_object(data, true).move_as_ok(); flags_ = obj->flags_ & ~(Flags::dbf_processed | Flags::dbf_moved_handle); id_ = create_block_id(obj->id_); prev_[0] = (flags_ & dbf_inited_prev_left) ? create_block_id(obj->prev_left_) : BlockIdExt{}; diff --git a/validator/block-handle.hpp b/validator/block-handle.hpp index 4e97f2b5..372c6140 100644 --- a/validator/block-handle.hpp +++ b/validator/block-handle.hpp @@ -517,7 +517,7 @@ struct BlockHandleImpl : public BlockHandleInterface { : id_(id), flags_(id_.is_masterchain() ? static_cast(dbf_masterchain) : 0) { get_thread_safe_counter().add(1); } - BlockHandleImpl(td::BufferSlice data); + BlockHandleImpl(td::Slice data); ~BlockHandleImpl() { LOG_CHECK(!need_flush()) << "flags=" << flags_; get_thread_safe_counter().add(-1); @@ -532,7 +532,7 @@ struct BlockHandleImpl : public BlockHandleInterface { return std::make_shared(id); } - static BlockHandle create(td::BufferSlice data) { + static BlockHandle create(td::Slice data) { return std::make_shared(std::move(data)); } }; diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 2a94c31d..2ea3b583 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -77,10 +77,20 @@ void ArchiveManager::update_handle(BlockHandle handle, td::Promise pro FileDescription *f; if (handle->handle_moved_to_archive()) { CHECK(handle->inited_unix_time()); + if (!handle->need_flush()) { + promise.set_value(td::Unit()); + return; + } f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), handle->id().seqno(), handle->unix_time(), handle->logical_time(), true); + if (!f) { + handle->flushed_upto(handle->version()); + promise.set_value(td::Unit()); + return; + } } else { f = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true); + CHECK(f); } td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::update_handle, std::move(handle), std::move(promise)); } @@ -540,7 +550,7 @@ void ArchiveManager::load_package(PackageId id) { } } - desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, db_root_); + desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_); get_file_map(id).emplace(id, std::move(desc)); } @@ -551,7 +561,6 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard auto it = f.find(id); if (it != f.end()) { if (it->second.deleted) { - CHECK(!force); return nullptr; } if (force && !id.temp) { @@ -574,7 +583,7 @@ ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard FileDescription desc{id, false}; td::mkdir(db_root_ + id.path()).ensure(); std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); - desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, db_root_); + desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_); if (!id.temp) { update_desc(desc, shard, seqno, ts, lt); } @@ -648,7 +657,11 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdF for (auto it = f.rbegin(); it != f.rend(); it++) { auto i = it->second.first_blocks.find(shard); if (i != it->second.first_blocks.end() && i->second.seqno <= seqno) { - return &it->second; + if (it->second.deleted) { + return nullptr; + } else { + return &it->second; + } } } return nullptr; @@ -660,7 +673,11 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(Shar for (auto it = f.rbegin(); it != f.rend(); it++) { auto i = it->second.first_blocks.find(shard); if (i != it->second.first_blocks.end() && i->second.ts <= ts) { - return &it->second; + if (it->second.deleted) { + return nullptr; + } else { + return &it->second; + } } } return nullptr; @@ -672,7 +689,11 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull for (auto it = f.rbegin(); it != f.rend(); it++) { auto i = it->second.first_blocks.find(shard); if (i != it->second.first_blocks.end() && i->second.lt <= lt) { - return &it->second; + if (it->second.deleted) { + return nullptr; + } else { + return &it->second; + } } } return nullptr; @@ -823,6 +844,14 @@ void ArchiveManager::start_up() { } } + v = index_->get("finalizedupto", value); + v.ensure(); + if (v.move_as_ok() == td::KeyValue::GetStatus::Ok) { + auto R = td::to_integer_safe(value); + R.ensure(); + finalized_up_to_ = R.move_as_ok(); + } + td::WalkPath::run(db_root_ + "/archive/states/", [&](td::CSlice fname, td::WalkPath::Type t) -> void { if (t == td::WalkPath::Type::NotDir) { LOG(ERROR) << "checking file " << fname; @@ -853,7 +882,7 @@ void ArchiveManager::start_up() { persistent_state_gc(FileHash::zero()); } -void ArchiveManager::run_gc(UnixTime ts) { +void ArchiveManager::run_gc(UnixTime ts, UnixTime archive_ttl) { auto p = get_temp_package_id_by_unixtime(ts); std::vector vec; for (auto &x : temp_files_) { @@ -863,13 +892,37 @@ void ArchiveManager::run_gc(UnixTime ts) { break; } } - if (vec.size() <= 1) { - return; - } - vec.resize(vec.size() - 1, PackageId::empty(false, true)); + if (vec.size() > 1) { + vec.resize(vec.size() - 1, PackageId::empty(false, true)); - for (auto &x : vec) { - delete_package(x, [](td::Unit) {}); + for (auto &x : vec) { + delete_package(x, [](td::Unit) {}); + } + } + vec.clear(); + + if (archive_ttl > 0) { + for (auto &f : files_) { + auto &desc = f.second; + if (desc.deleted) { + continue; + } + auto it = desc.first_blocks.find(ShardIdFull{masterchainId}); + if (it == desc.first_blocks.end()) { + continue; + } + if (it->second.ts < ts - archive_ttl) { + vec.push_back(f.first); + } + } + if (vec.size() > 1) { + vec.resize(vec.size() - 1, PackageId::empty(false, true)); + + for (auto &x : vec) { + LOG(ERROR) << "WARNING: deleting package " << x.id; + delete_package(x, [](td::Unit) {}); + } + } } } @@ -1049,6 +1102,102 @@ void ArchiveManager::set_async_mode(bool mode, td::Promise promise) { } } +void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { + index_->begin_transaction().ensure(); + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + for (auto &x : temp_files_) { + if (!x.second.deleted) { + td::actor::send_closure(x.second.file_actor_id(), &ArchiveSlice::destroy, ig.get_promise()); + x.second.file.release(); + } + } + temp_files_.clear(); + + { + auto it = key_files_.begin(); + while (it != key_files_.end()) { + if (it->first.id <= masterchain_seqno) { + td::actor::send_closure(it->second.file_actor_id(), &ArchiveSlice::truncate, masterchain_seqno, handle, + ig.get_promise()); + it++; + } else { + auto it2 = it; + it++; + td::actor::send_closure(it2->second.file_actor_id(), &ArchiveSlice::destroy, ig.get_promise()); + it2->second.file.release(); + index_ + ->erase(create_serialize_tl_object(it2->second.id.id, it2->second.id.key, + it2->second.id.temp) + .as_slice()) + .ensure(); + key_files_.erase(it2); + } + } + } + { + auto it = files_.begin(); + while (it != files_.end()) { + if (it->first.id <= masterchain_seqno) { + td::actor::send_closure(it->second.file_actor_id(), &ArchiveSlice::truncate, masterchain_seqno, handle, + ig.get_promise()); + it++; + } else { + auto it2 = it; + it++; + td::actor::send_closure(it2->second.file_actor_id(), &ArchiveSlice::destroy, ig.get_promise()); + it2->second.file.release(); + index_ + ->erase(create_serialize_tl_object(it2->second.id.id, it2->second.id.key, + it2->second.id.temp) + .as_slice()) + .ensure(); + files_.erase(it2); + } + } + } + { + std::vector t; + std::vector tk; + std::vector tt; + for (auto &e : files_) { + t.push_back(e.first.id); + } + for (auto &e : key_files_) { + tk.push_back(e.first.id); + } + for (auto &e : temp_files_) { + tt.push_back(e.first.id); + } + index_ + ->set(create_serialize_tl_object().as_slice(), + create_serialize_tl_object(std::move(t), std::move(tk), std::move(tt)) + .as_slice()) + .ensure(); + } + index_->commit_transaction().ensure(); + + { + auto it = perm_states_.begin(); + while (it != perm_states_.end()) { + int res = 0; + it->second.ref().visit(td::overloaded( + [&](const fileref::ZeroStateShort &x) { res = -1; }, + [&](const fileref::PersistentStateShort &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, + [&](const auto &obj) { res = 1; })); + if (res <= 0) { + it++; + } else { + auto it2 = it; + it++; + td::unlink(db_root_ + "/archive/states/" + it2->second.filename_short()).ignore(); + perm_states_.erase(it2); + } + } + } +} + } // namespace validator } // namespace ton diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 7d5f0947..dbe6e4d4 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -52,10 +52,10 @@ class ArchiveManager : public td::actor::Actor { void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); void check_zero_state(BlockIdExt block_id, td::Promise promise); - //void truncate(BlockSeqno masterchain_seqno, td::Promise promise); + void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise); //void truncate_continue(BlockSeqno masterchain_seqno, td::Promise promise); - void run_gc(UnixTime ts); + void run_gc(UnixTime ts, UnixTime archive_ttl); /* from LTDB */ void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise); @@ -104,6 +104,7 @@ class ArchiveManager : public td::actor::Actor { std::map files_; std::map key_files_; std::map temp_files_; + BlockSeqno finalized_up_to_{0}; bool async_mode_ = false; bool huge_transaction_started_ = false; td::uint32 huge_transaction_size_ = 0; diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 5b6fe157..5f1bc708 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -309,7 +309,7 @@ void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id, } } f = true; - auto G = fetch_tl_object(td::BufferSlice{value}, true); + auto G = fetch_tl_object(value, true); G.ensure(); auto g = G.move_as_ok(); if (compare_desc(*g.get()) > 0) { @@ -467,12 +467,18 @@ void ArchiveSlice::start_up() { R2 = kv_->get(PSTRING() << "status." << i, value); R2.ensure(); auto len = td::to_integer(value); + R2 = kv_->get(PSTRING() << "version." << i, value); + R2.ensure(); + td::uint32 ver = 0; + if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { + ver = td::to_integer(value); + } auto v = archive_id_ + slice_size_ * i; - add_package(v, len); + add_package(v, len, ver); } } else { auto len = td::to_integer(value); - add_package(archive_id_, len); + add_package(archive_id_, len, 0); } } else { if (!temp_ && !key_blocks_only_) { @@ -482,13 +488,15 @@ void ArchiveSlice::start_up() { kv_->set("slices", "1").ensure(); kv_->set("slice_size", td::to_string(slice_size_)).ensure(); kv_->set("status.0", "0").ensure(); + kv_->set("version.0", td::to_string(default_package_version())).ensure(); kv_->commit_transaction().ensure(); + add_package(archive_id_, 0, default_package_version()); } else { kv_->begin_transaction().ensure(); kv_->set("status", "0").ensure(); kv_->commit_transaction().ensure(); + add_package(archive_id_, 0, 0); } - add_package(archive_id_, 0); } } @@ -528,8 +536,12 @@ void ArchiveSlice::set_async_mode(bool mode, td::Promise promise) { } } -ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, std::string db_root) - : archive_id_(archive_id), key_blocks_only_(key_blocks_only), temp_(temp), db_root_(std::move(db_root)) { +ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root) + : archive_id_(archive_id) + , key_blocks_only_(key_blocks_only) + , temp_(temp) + , finalized_(finalized) + , db_root_(std::move(db_root)) { } td::Result ArchiveSlice::choose_package(BlockSeqno masterchain_seqno, bool force) { @@ -548,16 +560,17 @@ td::Result ArchiveSlice::choose_package(BlockSeqno begin_transaction(); kv_->set("slices", td::to_string(v + 1)).ensure(); kv_->set(PSTRING() << "status." << v, "0").ensure(); + kv_->set(PSTRING() << "version." << v, td::to_string(default_package_version())).ensure(); commit_transaction(); CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); - add_package(masterchain_seqno, 0); + add_package(masterchain_seqno, 0, default_package_version()); return &packages_[v]; } else { return &packages_[v]; } } -void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size) { +void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 version) { PackageId p_id{seqno, key_blocks_only_, temp_}; std::string path = PSTRING() << db_root_ << p_id.path() << p_id.name() << ".pack"; auto R = Package::open(path, false, true); @@ -565,9 +578,17 @@ void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size) { LOG(FATAL) << "failed to open/create archive '" << path << "': " << R.move_as_error(); return; } + auto idx = td::narrow_cast(packages_.size()); + if (finalized_) { + packages_.emplace_back(nullptr, td::actor::ActorOwn(), seqno, path, idx, version); + return; + } auto pack = std::make_shared(R.move_as_ok()); + if (version >= 1) { + pack->truncate(size).ensure(); + } auto writer = td::actor::create_actor("writer", pack); - packages_.emplace_back(std::move(pack), std::move(writer), seqno, path, 0); + packages_.emplace_back(std::move(pack), std::move(writer), seqno, path, idx, version); } namespace { @@ -609,6 +630,207 @@ void ArchiveSlice::destroy(td::Promise promise) { td::Timestamp::in(0.0)); } +BlockSeqno ArchiveSlice::max_masterchain_seqno() { + auto key = get_db_key_lt_desc(ShardIdFull{masterchainId}); + std::string value; + auto F = kv_->get(key, value); + F.ensure(); + if (F.move_as_ok() == td::KeyValue::GetStatus::NotFound) { + return 0; + } + auto G = fetch_tl_object(value, true); + G.ensure(); + auto g = G.move_as_ok(); + if (g->first_idx_ == g->last_idx_) { + return 0; + } + auto last_idx = g->last_idx_ - 1; + auto db_key = get_db_key_lt_el(ShardIdFull{masterchainId}, last_idx); + F = kv_->get(db_key, value); + F.ensure(); + CHECK(F.move_as_ok() == td::KeyValue::GetStatus::Ok); + auto E = fetch_tl_object(td::BufferSlice{value}, true); + E.ensure(); + auto e = E.move_as_ok(); + return e->id_->seqno_; +} + +void ArchiveSlice::delete_file(FileReference ref_id) { + std::string value; + auto R = kv_->get(ref_id.hash().to_hex(), value); + R.ensure(); + if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) { + return; + } + kv_->erase(ref_id.hash().to_hex()); +} + +void ArchiveSlice::delete_handle(ConstBlockHandle handle) { + delete_file(fileref::Proof{handle->id()}); + delete_file(fileref::ProofLink{handle->id()}); + delete_file(fileref::Block{handle->id()}); + kv_->erase(get_db_key_block_info(handle->id())); +} + +void ArchiveSlice::move_file(FileReference ref_id, Package *old_pack, Package *pack) { + LOG(DEBUG) << "moving " << ref_id.filename_short(); + std::string value; + auto R = kv_->get(ref_id.hash().to_hex(), value); + R.ensure(); + if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) { + return; + } + auto offset = td::to_integer(value); + auto V = old_pack->read(offset); + V.ensure(); + auto data = std::move(V.move_as_ok().second); + auto r = pack->append(ref_id.filename(), std::move(data), false); + kv_->set(ref_id.hash().to_hex(), td::to_string(r)); +} + +void ArchiveSlice::move_handle(ConstBlockHandle handle, Package *old_pack, Package *pack) { + move_file(fileref::Proof{handle->id()}, old_pack, pack); + move_file(fileref::ProofLink{handle->id()}, old_pack, pack); + move_file(fileref::Block{handle->id()}, old_pack, pack); +} + +bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx, + Package *pack) { + std::string value; + auto R = kv_->get(get_db_key_block_info(block_id), value); + R.ensure(); + CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok); + auto E = create_block_handle(value); + E.ensure(); + auto handle = E.move_as_ok(); + auto seqno = handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block(); + if (seqno > masterchain_seqno) { + delete_handle(std::move(handle)); + return false; + } + + auto S = choose_package(seqno, false); + S.ensure(); + auto p = S.move_as_ok(); + CHECK(p->idx <= cutoff_idx); + if (p->idx == cutoff_idx) { + move_handle(std::move(handle), p->package.get(), pack); + } + + return true; +} + +void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx, + Package *pack) { + auto key = get_db_key_lt_desc(shard); + std::string value; + auto F = kv_->get(key, value); + F.ensure(); + if (F.move_as_ok() == td::KeyValue::GetStatus::NotFound) { + return; + } + auto G = fetch_tl_object(value, true); + G.ensure(); + auto g = G.move_as_ok(); + if (g->first_idx_ == g->last_idx_) { + return; + } + + int new_last_idx = g->first_idx_; + for (int i = g->first_idx_; i < g->last_idx_; i++) { + auto db_key = get_db_key_lt_el(shard, i); + F = kv_->get(db_key, value); + F.ensure(); + CHECK(F.move_as_ok() == td::KeyValue::GetStatus::Ok); + auto E = fetch_tl_object(value, true); + E.ensure(); + auto e = E.move_as_ok(); + + if (truncate_block(masterchain_seqno, create_block_id(e->id_), cutoff_idx, pack)) { + CHECK(new_last_idx == i); + new_last_idx = i + 1; + } + } + + if (g->last_idx_ != new_last_idx) { + g->last_idx_ = new_last_idx; + kv_->set(key, serialize_tl_object(g, true)).ensure(); + } +} + +void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { + if (temp_ || archive_id_ > masterchain_seqno) { + destroy(std::move(promise)); + return; + } + LOG(INFO) << "TRUNCATE: slice " << archive_id_ << " maxseqno= " << max_masterchain_seqno() + << " truncate_upto=" << masterchain_seqno; + if (max_masterchain_seqno() <= masterchain_seqno) { + promise.set_value(td::Unit()); + return; + } + + auto cutoff = choose_package(masterchain_seqno, false); + cutoff.ensure(); + auto pack = cutoff.move_as_ok(); + CHECK(pack); + + auto pack_r = Package::open(pack->path + ".new", false, true); + pack_r.ensure(); + auto new_package = std::make_shared(pack_r.move_as_ok()); + new_package->truncate(0).ensure(); + + std::string value; + auto status_key = create_serialize_tl_object(); + auto R = kv_->get(status_key, value); + R.ensure(); + + auto F = fetch_tl_object(value, true); + F.ensure(); + auto f = F.move_as_ok(); + + kv_->begin_transaction().ensure(); + for (int i = 0; i < f->total_shards_; i++) { + auto shard_key = create_serialize_tl_object(i); + R = kv_->get(shard_key, value); + R.ensure(); + CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok); + + auto G = fetch_tl_object(value, true); + G.ensure(); + auto g = G.move_as_ok(); + + truncate_shard(masterchain_seqno, ShardIdFull{g->workchain_, static_cast(g->shard_)}, pack->idx, + new_package.get()); + } + + if (!sliced_mode_) { + kv_->set("status", td::to_string(new_package->size())).ensure(); + } else { + kv_->set(PSTRING() << "status." << pack->idx, td::to_string(new_package->size())).ensure(); + for (size_t i = pack->idx + 1; i < packages_.size(); i++) { + kv_->erase(PSTRING() << "status." << i); + kv_->erase(PSTRING() << "version." << i); + } + kv_->set("slices", td::to_string(pack->idx + 1)); + } + + pack->package = new_package; + pack->writer.reset(); + td::unlink(pack->path).ensure(); + td::rename(pack->path + ".new", pack->path).ensure(); + pack->writer = td::actor::create_actor("writer", new_package); + + for (auto idx = pack->idx + 1; idx < packages_.size(); idx++) { + td::unlink(packages_[idx].path).ensure(); + } + packages_.erase(packages_.begin() + pack->idx + 1); + + kv_->commit_transaction().ensure(); + + promise.set_value(td::Unit()); +} + } // namespace validator } // namespace ton diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index e066288a..487115fe 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -73,7 +73,7 @@ class PackageWriter : public td::actor::Actor { class ArchiveSlice : public td::actor::Actor { public: - ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, std::string db_root); + ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root); void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise); @@ -97,6 +97,7 @@ class ArchiveSlice : public td::actor::Actor { void start_up() override; void destroy(td::Promise promise); + void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise); void begin_transaction(); void commit_transaction(); @@ -117,6 +118,7 @@ class ArchiveSlice : public td::actor::Actor { bool key_blocks_only_; bool temp_; + bool finalized_; bool destroyed_ = false; bool async_mode_ = false; @@ -130,19 +132,38 @@ class ArchiveSlice : public td::actor::Actor { struct PackageInfo { PackageInfo(std::shared_ptr package, td::actor::ActorOwn writer, BlockSeqno id, - std::string path, td::uint32 idx) - : package(std::move(package)), writer(std ::move(writer)), id(id), path(std::move(path)), idx(idx) { + std::string path, td::uint32 idx, td::uint32 version) + : package(std::move(package)) + , writer(std ::move(writer)) + , id(id) + , path(std::move(path)) + , idx(idx) + , version(version) { } std::shared_ptr package; td::actor::ActorOwn writer; BlockSeqno id; std::string path; td::uint32 idx; + td::uint32 version; }; std::vector packages_; td::Result choose_package(BlockSeqno masterchain_seqno, bool force); - void add_package(BlockSeqno masterchain_seqno, td::uint64 size); + void add_package(BlockSeqno masterchain_seqno, td::uint64 size, td::uint32 version); + void truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx, Package *pack); + bool truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx, Package *pack); + + void delete_handle(ConstBlockHandle handle); + void delete_file(FileReference ref_id); + void move_handle(ConstBlockHandle handle, Package *old_pack, Package *pack); + void move_file(FileReference ref_id, Package *old_pack, Package *pack); + + BlockSeqno max_masterchain_seqno(); + + static constexpr td::uint32 default_package_version() { + return 1; + } }; } // namespace validator diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 189e1a13..6b1d99d8 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -44,9 +44,9 @@ void CellDbIn::start_up() { auto empty = get_empty_key_hash(); if (get_block(empty).is_error()) { DbEntry e{get_empty_key(), empty, empty, RootHash::zero()}; - cell_db_->begin_transaction().ensure(); + cell_db_->begin_write_batch().ensure(); set_block(empty, std::move(e)); - cell_db_->commit_transaction().ensure(); + cell_db_->commit_write_batch().ensure(); } last_gc_ = empty; } @@ -89,12 +89,12 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi boc_->inc(cell); boc_->prepare_commit().ensure(); vm::CellStorer stor{*cell_db_.get()}; - cell_db_->begin_transaction().ensure(); + cell_db_->begin_write_batch().ensure(); boc_->commit(stor).ensure(); set_block(empty, std::move(E)); set_block(D.prev, std::move(P)); set_block(key_hash, std::move(D)); - cell_db_->commit_transaction().ensure(); + cell_db_->commit_write_batch().ensure(); boc_->set_loader(std::make_unique(cell_db_->snapshot())).ensure(); td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); @@ -181,12 +181,12 @@ void CellDbIn::gc_cont2(BlockHandle handle) { boc_->dec(cell); boc_->prepare_commit().ensure(); vm::CellStorer stor{*cell_db_.get()}; - cell_db_->begin_transaction().ensure(); + cell_db_->begin_write_batch().ensure(); boc_->commit(stor).ensure(); cell_db_->erase(get_key(last_gc_)).ensure(); set_block(F.prev, std::move(P)); set_block(F.next, std::move(N)); - cell_db_->commit_transaction().ensure(); + cell_db_->commit_write_batch().ensure(); alarm_timestamp() = td::Timestamp::now(); boc_->set_loader(std::make_unique(cell_db_->snapshot())).ensure(); diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index ea417643..ef88ad5f 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -409,10 +409,13 @@ void RootDb::prepare_stats(td::Promise state, td::Promise promise) { +void RootDb::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) { td::MultiPromise mp; auto ig = mp.init_guard(); ig.add_promise(std::move(promise)); + + td::actor::send_closure(archive_db_, &ArchiveManager::truncate, seqno, handle, ig.get_promise()); + td::actor::send_closure(state_db_, &StateDb::truncate, seqno, handle, ig.get_promise()); } void RootDb::add_key_block_proof(td::Ref proof, td::Promise promise) { @@ -483,8 +486,8 @@ void RootDb::set_async_mode(bool mode, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise)); } -void RootDb::run_gc(UnixTime ts) { - td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, ts); +void RootDb::run_gc(UnixTime ts, UnixTime archive_ttl) { + td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, ts, archive_ttl); } } // namespace validator diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 380c568a..c4d642e7 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -34,8 +34,8 @@ namespace validator { class RootDb : public Db { public: enum class Flags : td::uint32 { f_started = 1, f_ready = 2, f_switched = 4, f_archived = 8 }; - RootDb(td::actor::ActorId validator_manager, std::string root_path, td::uint32 depth) - : validator_manager_(validator_manager), root_path_(std::move(root_path)), depth_(depth) { + RootDb(td::actor::ActorId validator_manager, std::string root_path) + : validator_manager_(validator_manager), root_path_(std::move(root_path)) { } void start_up() override; @@ -84,7 +84,8 @@ class RootDb : public Db { void apply_block(BlockHandle handle, td::Promise promise) override; void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) override; void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise promise) override; - void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) override; + void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, + td::Promise promise) override; void update_init_masterchain_block(BlockIdExt block, td::Promise promise) override; void get_init_masterchain_block(td::Promise promise) override; @@ -113,7 +114,7 @@ class RootDb : public Db { void prepare_stats(td::Promise>> promise) override; - void truncate(td::Ref state, td::Promise promise) override; + void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) override; void add_key_block_proof(td::Ref proof, td::Promise promise) override; void add_key_block_proof_link(td::Ref proof_link, td::Promise promise) override; @@ -127,13 +128,12 @@ class RootDb : public Db { td::Promise promise) override; void set_async_mode(bool mode, td::Promise promise) override; - void run_gc(UnixTime ts) override; + void run_gc(UnixTime ts, UnixTime archive_ttl) override; private: td::actor::ActorId validator_manager_; std::string root_path_; - td::uint32 depth_; td::actor::ActorOwn cell_db_; td::actor::ActorOwn state_db_; diff --git a/validator/db/statedb.cpp b/validator/db/statedb.cpp index 14764929..5d49ae2b 100644 --- a/validator/db/statedb.cpp +++ b/validator/db/statedb.cpp @@ -29,11 +29,11 @@ namespace validator { void StateDb::update_init_masterchain_block(BlockIdExt block, td::Promise promise) { auto key = create_hash_tl_object(); - kv_->begin_transaction().ensure(); + kv_->begin_write_batch().ensure(); kv_->set(key.as_slice(), create_serialize_tl_object(create_tl_block_id(block)).as_slice()) .ensure(); - kv_->commit_transaction().ensure(); + kv_->commit_write_batch().ensure(); promise.set_value(td::Unit()); } @@ -59,11 +59,11 @@ void StateDb::get_init_masterchain_block(td::Promise promise) { void StateDb::update_gc_masterchain_block(BlockIdExt block, td::Promise promise) { auto key = create_hash_tl_object(); - kv_->begin_transaction().ensure(); + kv_->begin_write_batch().ensure(); kv_->set(key.as_slice(), create_serialize_tl_object(create_tl_block_id(block)).as_slice()) .ensure(); - kv_->commit_transaction().ensure(); + kv_->commit_write_batch().ensure(); promise.set_value(td::Unit()); } @@ -89,11 +89,11 @@ void StateDb::get_gc_masterchain_block(td::Promise promise) { void StateDb::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) { auto key = create_hash_tl_object(); - kv_->begin_transaction().ensure(); + kv_->begin_write_batch().ensure(); kv_->set(key.as_slice(), create_serialize_tl_object(create_tl_block_id(masterchain_block_id))) .ensure(); - kv_->commit_transaction().ensure(); + kv_->commit_write_batch().ensure(); promise.set_value(td::Unit()); } @@ -120,10 +120,10 @@ void StateDb::update_destroyed_validator_sessions(std::vector promise) { auto key = create_hash_tl_object(); - kv_->begin_transaction().ensure(); + kv_->begin_write_batch().ensure(); kv_->set(key.as_slice(), create_serialize_tl_object(std::move(sessions))) .ensure(); - kv_->commit_transaction().ensure(); + kv_->commit_write_batch().ensure(); promise.set_value(td::Unit()); } @@ -153,9 +153,9 @@ void StateDb::update_async_serializer_state(AsyncSerializerState state, td::Prom create_tl_block_id(state.last_block_id), create_tl_block_id(state.last_written_block_id), state.last_written_block_ts); - kv_->begin_transaction().ensure(); + kv_->begin_write_batch().ensure(); kv_->set(key.as_slice(), value.as_slice()).ensure(); - kv_->commit_transaction().ensure(); + kv_->commit_write_batch().ensure(); promise.set_value(td::Unit()); } @@ -188,9 +188,9 @@ void StateDb::update_hardforks(std::vector blocks, td::Promisebegin_transaction().ensure(); + kv_->begin_write_batch().ensure(); kv_->set(key.as_slice(), create_serialize_tl_object(std::move(vec))).ensure(); - kv_->commit_transaction(); + kv_->commit_write_batch(); promise.set_value(td::Unit()); } @@ -232,14 +232,95 @@ void StateDb::start_up() { auto f = F.move_as_ok(); CHECK(f->version_ == 2); } else { - kv_->begin_transaction().ensure(); + kv_->begin_write_batch().ensure(); kv_->set(create_serialize_tl_object(), create_serialize_tl_object(2)) .ensure(); - kv_->commit_transaction().ensure(); + kv_->commit_write_batch().ensure(); } } +void StateDb::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { + { + auto key = create_hash_tl_object(); + + std::string value; + auto R = kv_->get(key.as_slice(), value); + R.ensure(); + + if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(value, true); + F.ensure(); + auto obj = F.move_as_ok(); + if (static_cast(obj->last_->seqno_) > masterchain_seqno) { + CHECK(handle); + CHECK(handle->inited_unix_time()); + obj->last_ = create_tl_block_id(handle->id()); + obj->last_ts_ = handle->unix_time(); + kv_->begin_write_batch().ensure(); + kv_->set(key.as_slice(), serialize_tl_object(obj, true)).ensure(); + kv_->commit_write_batch().ensure(); + } + } + } + { + auto key = create_hash_tl_object(); + + std::string value; + auto R = kv_->get(key.as_slice(), value); + R.ensure(); + + if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(td::BufferSlice{value}, true); + F.ensure(); + auto obj = F.move_as_ok(); + if (static_cast(obj->block_->seqno_) > masterchain_seqno) { + CHECK(handle); + obj->block_ = create_tl_block_id(handle->id()); + kv_->begin_write_batch().ensure(); + kv_->set(key.as_slice(), serialize_tl_object(obj, true)).ensure(); + kv_->commit_write_batch().ensure(); + } + } + } + { + auto key = create_hash_tl_object(); + + std::string value; + auto R = kv_->get(key.as_slice(), value); + R.ensure(); + + if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(td::BufferSlice{value}, true); + F.ensure(); + auto obj = F.move_as_ok(); + CHECK(static_cast(obj->block_->seqno_) <= masterchain_seqno); + } + } + { + auto key = create_hash_tl_object(); + + std::string value; + auto R = kv_->get(key.as_slice(), value); + R.ensure(); + + if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(td::BufferSlice{value}, true); + F.ensure(); + auto obj = F.move_as_ok(); + if (static_cast(obj->block_->seqno_) > masterchain_seqno) { + CHECK(handle); + obj->block_ = create_tl_block_id(handle->id()); + kv_->begin_write_batch().ensure(); + kv_->set(key.as_slice(), serialize_tl_object(obj, true)).ensure(); + kv_->commit_write_batch().ensure(); + } + } + } + + promise.set_value(td::Unit()); +} + } // namespace validator } // namespace ton diff --git a/validator/db/statedb.hpp b/validator/db/statedb.hpp index 1ad411c4..75382d61 100644 --- a/validator/db/statedb.hpp +++ b/validator/db/statedb.hpp @@ -56,6 +56,7 @@ class StateDb : public td::actor::Actor { StateDb(td::actor::ActorId root_db, std::string path); void start_up() override; + void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise); private: using KeyType = td::Bits256; diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 1b4e03da..56137fc3 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -20,6 +20,7 @@ #include "validator/fabric.h" #include "ton/ton-io.hpp" #include "common/checksum.h" +#include "common/delay.h" namespace ton { @@ -95,8 +96,6 @@ void WaitBlockState::start() { }); td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, handle_->id().file_hash, std::move(P)); } else if (handle_->id().id.seqno == 0) { - // do not try to download full chain - // download state + proof_link only auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockState::failed_to_get_state_from_net, @@ -107,21 +106,19 @@ void WaitBlockState::start() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_zero_state_request, handle_->id(), priority_, std::move(P)); - } else if (block_.is_null()) { - // download block and then prev state, not in reverse only - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { - td::actor::send_closure(SelfId, &WaitBlockState::failed_to_get_block_data, - R.move_as_error_prefix("block wait error: ")); + delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::start); }, td::Timestamp::in(0.1)); } else { - td::actor::send_closure(SelfId, &WaitBlockState::got_block_data, R.move_as_ok()); + td::actor::send_closure(SelfId, &WaitBlockState::got_proof_link, R.move_as_ok()); } }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_data, handle_, priority_, timeout_, std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), priority_, + std::move(P)); } else if (prev_state_.is_null()) { CHECK(handle_->inited_proof() || handle_->inited_proof_link()); - CHECK(handle_->received()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockState::failed_to_get_prev_state, @@ -133,6 +130,28 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::wait_prev_block_state, handle_, priority_, timeout_, std::move(P)); + } else if (handle_->id().is_masterchain() && !handle_->inited_proof()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { + if (R.is_error()) { + delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::start); }, td::Timestamp::in(0.1)); + } else { + td::actor::send_closure(SelfId, &WaitBlockState::got_proof, R.move_as_ok()); + } + }); + + td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_request, handle_->id(), priority_, + std::move(P)); + } else if (block_.is_null()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockState::failed_to_get_block_data, + R.move_as_error_prefix("block wait error: ")); + } else { + td::actor::send_closure(SelfId, &WaitBlockState::got_block_data, R.move_as_ok()); + } + }); + + td::actor::send_closure(manager_, &ValidatorManager::wait_block_data, handle_, priority_, timeout_, std::move(P)); } else { apply(); } @@ -152,6 +171,39 @@ void WaitBlockState::got_prev_state(td::Ref state) { start(); } +void WaitBlockState::got_proof_link(td::BufferSlice data) { + auto R = create_proof_link(handle_->id(), std::move(data)); + if (R.is_error()) { + LOG(INFO) << "received bad proof link: " << R.move_as_error(); + start(); + return; + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + auto h = R.move_as_ok(); + CHECK(h->inited_prev()); + td::actor::send_closure(SelfId, &WaitBlockState::start); + } else { + LOG(INFO) << "received bad proof link: " << R.move_as_error(); + td::actor::send_closure(SelfId, &WaitBlockState::start); + } + }); + run_check_proof_link_query(handle_->id(), R.move_as_ok(), manager_, timeout_, std::move(P)); +} + +void WaitBlockState::got_proof(td::BufferSlice data) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &WaitBlockState::start); + } else { + LOG(INFO) << "received bad proof link: " << R.move_as_error(); + td::actor::send_closure(SelfId, &WaitBlockState::start); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::validate_block_proof, handle_->id(), std::move(data), + std::move(P)); +} + void WaitBlockState::failed_to_get_block_data(td::Status reason) { if (reason.code() == ErrorCode::notready) { start(); diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index b2390917..a93f2971 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -54,6 +54,8 @@ class WaitBlockState : public td::actor::Actor { void got_state_from_net(td::BufferSlice data); void failed_to_get_zero_state(); void failed_to_get_state_from_net(td::Status reason); + void got_proof_link(td::BufferSlice data); + void got_proof(td::BufferSlice data); void apply(); void written_state(td::Ref upd_state); void written_state_file(); diff --git a/validator/fabric.h b/validator/fabric.h index 779c8fee..a10251dd 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -25,8 +25,7 @@ namespace ton { namespace validator { -td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, - td::uint32 depth); +td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_); td::actor::ActorOwn create_liteserver_cache_actor(td::actor::ActorId manager, std::string db_root); @@ -38,6 +37,7 @@ td::Result> create_signature_set(td::BufferSlice sig_ td::Result> create_shard_state(BlockIdExt block_id, td::BufferSlice data); td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); td::Result create_block_handle(td::BufferSlice data); +td::Result create_block_handle(td::Slice data); td::Result create_temp_block_handle(td::BufferSlice data); BlockHandle create_empty_block_handle(BlockIdExt id); td::Result> create_ext_message(td::BufferSlice data); @@ -76,6 +76,9 @@ void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& m std::vector prev, Ed25519_PublicKey local_id, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); +void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, + td::actor::ActorId manager, td::Timestamp timeout, + td::Promise promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index 31aa7420..a5c4d9a1 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -262,9 +262,9 @@ bool AcceptBlockQuery::create_new_proof() { } else { // FAKE vm::CellBuilder cb2; if (!(cb2.store_long_bool(0x11, 8) // block_signatures#11 - && cb2.store_long_bool(validator_set_->get_validator_set_hash(), + && cb2.store_long_bool(validator_set_.not_null() ? validator_set_->get_validator_set_hash() : 0, 32) // validator_info$_ validator_set_hash_short:uint32 - && cb2.store_long_bool(validator_set_->get_catchain_seqno(), + && cb2.store_long_bool(validator_set_.not_null() ? validator_set_->get_catchain_seqno() : 0, 32) // validator_set_ts:uint32 = ValidatorInfo && cb2.store_long_bool(0, 32) // sig_count:uint32 && cb2.store_long_bool(0, 64) // sig_weight:uint32 @@ -355,7 +355,7 @@ void AcceptBlockQuery::start_up() { VLOG(VALIDATOR_DEBUG) << "start_up()"; alarm_timestamp() = timeout_; - if (validator_set_.is_null()) { + if (!is_fork_ && validator_set_.is_null()) { fatal_error("no real ValidatorSet passed to AcceptBlockQuery"); return; } diff --git a/validator/impl/check-proof.cpp b/validator/impl/check-proof.cpp index 1bf8e276..30a13c08 100644 --- a/validator/impl/check-proof.cpp +++ b/validator/impl/check-proof.cpp @@ -313,8 +313,8 @@ void CheckProof::start_up() { return; } - td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, id_, - true, [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, id_, true, + [SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &CheckProof::abort_query, R.move_as_error()); } else { @@ -344,8 +344,8 @@ void CheckProof::got_block_handle(BlockHandle handle) { process_masterchain_state(); return; } - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, prev_[0], priority(), - timeout_, [SelfId = actor_id(this)](td::Result> R) { + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, prev_[0], priority(), timeout_, + [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &CheckProof::got_masterchain_state, td::Ref{R.move_as_ok()}); @@ -440,8 +440,8 @@ void CheckProof::check_signatures(Ref s) { if (handle_) { got_block_handle_2(handle_); } else { - td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, id_, - true, [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, id_, true, + [SelfId = actor_id(this)](td::Result R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &CheckProof::got_block_handle_2, R.move_as_ok()); }); @@ -466,7 +466,10 @@ void CheckProof::got_block_handle_2(BlockHandle handle) { }); if (skip_check_signatures_) { // do not save proof if we skipped signatures - handle_->flush(manager_, handle_, std::move(P)); + auto proof = Ref(proof_); + CHECK(proof.not_null()); + td::actor::send_closure_later(manager_, &ValidatorManager::set_block_proof, handle_, std::move(proof), + std::move(P)); } else if (is_proof()) { auto proof = Ref(proof_); CHECK(proof.not_null()); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index b1a7a034..87aa5494 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -63,6 +63,7 @@ class Collator final : public td::actor::Actor { bool inbound_queues_empty_{false}; bool libraries_changed_{false}; bool prev_key_block_exists_{false}; + bool is_hardfork_{false}; UnixTime min_ts; BlockIdExt min_mc_block_id; std::vector prev_blocks; @@ -85,9 +86,9 @@ class Collator final : public td::actor::Actor { static constexpr bool shard_splitting_enabled = true; public: - Collator(ShardIdFull shard, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, - Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise); + Collator(ShardIdFull shard, bool is_hardfork, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, + std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, + td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); ~Collator() override = default; bool is_busy() const { return busy_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 0550d609..5f1c6b43 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -54,11 +54,12 @@ static inline bool dbg(int c) { return true; } -Collator::Collator(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, +Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) : shard_(shard) + , is_hardfork_(is_hardfork) , min_ts(min_ts) , min_mc_block_id{min_masterchain_block_id} , prev_blocks(std::move(prev)) @@ -75,6 +76,9 @@ void Collator::start_up() { if (prev_blocks.size() > 1) { LOG(DEBUG) << "Previous block #2 is " << prev_blocks.at(1).to_str(); } + if (is_hardfork_ && workchain() == masterchainId) { + is_key_block_ = true; + } // 1. check validity of parameters, especially prev_blocks, shard and min_mc_block_id if (workchain() != ton::masterchainId && workchain() != ton::basechainId) { fatal_error(-667, "can create block candidates only for masterchain (-1) and base workchain (0)"); @@ -162,12 +166,26 @@ void Collator::start_up() { // 2. learn latest masterchain state and block id LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { - LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, - std::move(res)); - }); + if (!is_hardfork_) { + td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, + [self = get_self()](td::Result, BlockIdExt>> res) { + LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, + std::move(res)); + }); + } else { + td::actor::send_closure_later( + manager, &ValidatorManager::get_shard_state_from_db_short, min_mc_block_id, + [self = get_self(), block_id = min_mc_block_id](td::Result> res) { + LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; + if (res.is_error()) { + td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, res.move_as_error()); + } else { + td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, + std::make_pair(Ref(res.move_as_ok()), block_id)); + } + }); + } } // 3. load previous block(s) and corresponding state(s) prev_states.resize(prev_blocks.size()); @@ -195,16 +213,21 @@ void Collator::start_up() { }); } } + if (is_hardfork_) { + LOG(WARNING) << "generating a hardfork block"; + } // 4. load external messages - LOG(DEBUG) << "sending get_external_messages() query to Manager"; - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::get_external_messages, shard_, - [self = get_self()](td::Result>> res) -> void { - LOG(DEBUG) << "got answer to get_external_messages() query"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_external_messages, - std::move(res)); - }); - if (is_masterchain()) { + if (!is_hardfork_) { + LOG(DEBUG) << "sending get_external_messages() query to Manager"; + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::get_external_messages, shard_, + [self = get_self()](td::Result>> res) -> void { + LOG(DEBUG) << "got answer to get_external_messages() query"; + td::actor::send_closure_later( + std::move(self), &Collator::after_get_external_messages, std::move(res)); + }); + } + if (is_masterchain() && !is_hardfork_) { // 5. load shard block info messages LOG(DEBUG) << "sending get_shard_blocks() query to Manager"; ++pending; @@ -487,6 +510,7 @@ void Collator::after_get_shard_blocks(td::Resultget_vert_seqno(); + vert_seqno_ = config_->get_vert_seqno() + (is_hardfork_ ? 1 : 0); LOG(DEBUG) << "vertical seqno (vert_seqno) is " << vert_seqno_; auto limits = config_->get_block_limits(is_masterchain()); if (limits.is_error()) { @@ -549,6 +573,9 @@ bool Collator::unpack_last_mc_state() { } bool Collator::check_cur_validator_set() { + if (is_hardfork_) { + return true; + } CatchainSeqno cc_seqno = 0; auto nodes = config_->compute_validator_set_cc(shard_, now_, &cc_seqno); if (nodes.empty()) { @@ -3083,12 +3110,16 @@ bool Collator::create_mc_state_extra() { auto cfg_smc_config = cfg_res.move_as_ok(); CHECK(cfg_smc_config.not_null()); vm::Dictionary cfg_dict{cfg_smc_config, 32}; + bool ignore_cfg_changes = false; + Ref cfg0; if (!block::valid_config_data(cfg_smc_config, config_addr, true, true, old_mparams_)) { block::gen::t_Hashmap_32_Ref_Cell.print_ref(std::cerr, cfg_smc_config); - return fatal_error("configuration smart contract "s + config_addr.to_hex() + - " contains an invalid configuration in its data"); + LOG(ERROR) << "configuration smart contract "s + config_addr.to_hex() + + " contains an invalid configuration in its data, IGNORING CHANGES"; + ignore_cfg_changes = true; + } else { + cfg0 = cfg_dict.lookup_ref(td::BitArray<32>(1 - 1)); } - Ref cfg0 = cfg_dict.lookup_ref(td::BitArray<32>(1 - 1)); bool changed_cfg = false; if (cfg0.not_null()) { ton::StdSmcAddress new_config_addr; @@ -3101,7 +3132,11 @@ bool Collator::create_mc_state_extra() { changed_cfg = true; } } - if (block::important_config_parameters_changed(cfg_smc_config, state_extra.config->prefetch_ref()) || changed_cfg) { + if (ignore_cfg_changes) { + LOG(ERROR) << "configuration changes ignored"; + return fatal_error("attempting to install invalid new configuration"); + } else if (block::important_config_parameters_changed(cfg_smc_config, state_extra.config->prefetch_ref()) || + changed_cfg) { LOG(WARNING) << "global configuration changed, updating"; vm::CellBuilder cb; CHECK(cb.store_bits_bool(config_addr) && cb.store_ref_bool(cfg_smc_config)); @@ -3625,6 +3660,7 @@ bool Collator::store_master_ref(vm::CellBuilder& cb) { bool Collator::update_processed_upto() { auto ref_mc_seqno = is_masterchain() ? new_block_seqno : prev_mc_block_seqno; + update_min_mc_seqno(ref_mc_seqno); if (last_proc_int_msg_.first) { if (!processed_upto_->insert(ref_mc_seqno, last_proc_int_msg_.first, last_proc_int_msg_.second.cbits())) { return fatal_error("cannot update our ProcessedUpto to reflect processed inbound message"); @@ -3694,8 +3730,8 @@ bool Collator::compute_total_balance() { bool Collator::create_block_info(Ref& block_info) { vm::CellBuilder cb, cb2; bool mc = is_masterchain(); - td::uint32 val_hash = validator_set_->get_validator_set_hash(); - CatchainSeqno cc_seqno = validator_set_->get_catchain_seqno(); + td::uint32 val_hash = is_hardfork_ ? 0 : validator_set_->get_validator_set_hash(); + CatchainSeqno cc_seqno = is_hardfork_ ? 0 : validator_set_->get_catchain_seqno(); return cb.store_long_bool(0x9bc7a987, 32) // block_info#9bc7a987 && cb.store_long_bool(0, 32) // version:uint32 && cb.store_bool_bool(!mc) // not_master:(## 1) @@ -3705,7 +3741,8 @@ bool Collator::create_block_info(Ref& block_info) { && cb.store_bool_bool(want_split_) // want_split:Bool && cb.store_bool_bool(want_merge_) // want_merge:Bool && cb.store_bool_bool(is_key_block_) // key_block:Bool - && cb.store_long_bool((int)report_version_, 9) // vert_seqno_incr:(## 1) flags:(## 8) + && cb.store_bool_bool(is_hardfork_) // vert_seqno_incr:(## 1) + && cb.store_long_bool((int)report_version_, 8) // flags:(## 8) && cb.store_long_bool(new_block_seqno, 32) // seq_no:# && cb.store_long_bool(vert_seqno_, 32) // vert_seq_no:# && block::ShardId{shard_}.serialize(cb) // shard:ShardIdent @@ -3721,6 +3758,9 @@ bool Collator::create_block_info(Ref& block_info) { && cb.store_builder_ref_bool(std::move(cb2)))) // .. ^BlkMasterInfo && store_prev_blk_ref(cb2, after_merge_) // prev_ref:.. && cb.store_builder_ref_bool(std::move(cb2)) // .. ^(PrevBlkInfo after_merge) + && (!is_hardfork_ || // prev_vert_ref:vert_seqno_incr?.. + (store_master_ref(cb2) // + && cb.store_builder_ref_bool(std::move(cb2)))) // .. ^(BlkPrevInfo 0) && cb.finalize_to(block_info); } diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 073a24a6..ac296eb4 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -39,9 +39,8 @@ namespace ton { namespace validator { -td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, - td::uint32 depth) { - return td::actor::create_actor("db", manager, db_root_, depth); +td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_) { + return td::actor::create_actor("db", manager, db_root_); } td::actor::ActorOwn create_liteserver_cache_actor(td::actor::ActorId manager, @@ -93,7 +92,11 @@ td::Result> create_shard_state(BlockIdExt block_id, td::Ref< } td::Result create_block_handle(td::BufferSlice data) { - return ton::validator::BlockHandleImpl::create(std::move(data)); + return ton::validator::BlockHandleImpl::create(data.as_slice()); +} + +td::Result create_block_handle(td::Slice data) { + return ton::validator::BlockHandleImpl::create(data); } td::Result create_temp_block_handle(td::BufferSlice data) { @@ -208,9 +211,24 @@ void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& m seqno = p.seqno(); } } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, min_ts, - min_masterchain_block_id, std::move(prev), std::move(validator_set), collator_id, - std::move(manager), timeout, std::move(promise)) + td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false, + min_ts, min_masterchain_block_id, std::move(prev), std::move(validator_set), + collator_id, std::move(manager), timeout, std::move(promise)) + .release(); +} + +void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, + td::actor::ActorId manager, td::Timestamp timeout, + td::Promise promise) { + BlockSeqno seqno = 0; + for (auto& p : prev) { + if (p.seqno() > seqno) { + seqno = p.seqno(); + } + } + td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, 0, + min_masterchain_block_id, std::move(prev), td::Ref{}, + Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise)) .release(); } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 0912de88..9ccdd263 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -32,6 +32,7 @@ #include "block/block.h" #include "block/block-parse.h" #include "block/block-auto.h" +#include "block/check-proof.h" #include "vm/dict.h" #include "vm/cells/MerkleProof.h" #include "vm/vm.h" @@ -160,10 +161,10 @@ void LiteQuery::start_up() { static_cast((q.mode_ & 128) ? (q.after_->lt_) : 0)); }, [&](lite_api::liteServer_getConfigParams& q) { - this->perform_getConfigParams(ton::create_block_id(q.id_), (q.mode_ & 0xfff) | 0x1000, q.param_list_); + this->perform_getConfigParams(ton::create_block_id(q.id_), (q.mode_ & 0xffff) | 0x10000, q.param_list_); }, [&](lite_api::liteServer_getConfigAll& q) { - this->perform_getConfigParams(ton::create_block_id(q.id_), (q.mode_ & 0xfff) | 0x2000); + this->perform_getConfigParams(ton::create_block_id(q.id_), (q.mode_ & 0xffff) | 0x20000); }, [&](lite_api::liteServer_getBlockProof& q) { this->perform_getBlockProof(ton::create_block_id(q.known_block_), @@ -204,7 +205,7 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { } td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [ Self = actor_id(this), mode ](td::Result, BlockIdExt>> res) { + [Self = actor_id(this), mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -242,7 +243,7 @@ void LiteQuery::perform_getBlock(BlockIdExt blkid) { return; } td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -268,7 +269,7 @@ void LiteQuery::perform_getBlockHeader(BlockIdExt blkid, int mode) { return; } td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [ Self = actor_id(this), blkid, mode ](td::Result> res) { + [Self = actor_id(this), blkid, mode](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -383,7 +384,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { } if (blkid.id.seqno) { td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -393,7 +394,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { }); } else { td::actor::send_closure_later(manager_, &ValidatorManager::get_zero_state, blkid, - [ Self = actor_id(this), blkid ](td::Result res) { + [Self = actor_id(this), blkid](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -452,7 +453,7 @@ bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); @@ -478,7 +479,7 @@ bool LiteQuery::request_mc_proof(BlockIdExt blkid, int mode) { ++pending_; td::actor::send_closure( manager_, &ValidatorManager::get_key_block_proof, blkid, - [ Self = actor_id(this), manager = manager_, blkid, mode ](td::Result R) { + [Self = actor_id(this), manager = manager_, blkid, mode](td::Result R) { if (R.is_ok()) { auto proof = create_proof(blkid, R.move_as_ok()); proof.ensure(); @@ -510,7 +511,7 @@ bool LiteQuery::request_mc_block_state(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); @@ -541,7 +542,7 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); @@ -563,7 +564,7 @@ bool LiteQuery::request_block_data(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); @@ -586,7 +587,7 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) { if (blkid.is_masterchain()) { td::actor::send_closure( manager_, &ValidatorManager::get_key_block_proof_link, blkid, - [ Self = actor_id(this), manager = manager_, blkid ](td::Result R) { + [Self = actor_id(this), manager = manager_, blkid](td::Result R) { if (R.is_ok()) { auto proof = create_proof(blkid, R.move_as_ok()); proof.ensure(); @@ -608,7 +609,7 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) { } else { td::actor::send_closure_later( manager_, &ValidatorManager::get_block_proof_link_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : ")); @@ -634,7 +635,7 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_zero_state, blkid, - [ Self = actor_id(this), blkid ](td::Result res) { + [Self = actor_id(this), blkid](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : ")); @@ -679,7 +680,7 @@ void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, LOG(INFO) << "sending a get_top_masterchain_state_block query to manager"; td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [Self = actor_id(this)](td::Result, BlockIdExt>> res)->void { + [Self = actor_id(this)](td::Result, BlockIdExt>> res) -> void { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -781,7 +782,7 @@ void LiteQuery::got_mc_block_state(BlockIdExt blkid, Ref state) { void LiteQuery::got_block_data(BlockIdExt blkid, Ref data) { LOG(INFO) << "obtained data for getBlock(" << blkid.to_str() << ") needed by a liteserver query"; CHECK(data.not_null()); - block_ = std::move(data); + block_ = Ref(std::move(data)); CHECK(block_.not_null()); CHECK(blkid == blk_id_); dec_pending(); @@ -790,7 +791,7 @@ void LiteQuery::got_block_data(BlockIdExt blkid, Ref data) { void LiteQuery::got_mc_block_data(BlockIdExt blkid, Ref data) { LOG(INFO) << "obtained data for getBlock(" << blkid.to_str() << ") needed by a liteserver query"; CHECK(data.not_null()); - mc_block_ = std::move(data); + mc_block_ = Ref(std::move(data)); CHECK(mc_block_.not_null()); CHECK(blkid == base_blk_id_); dec_pending(); @@ -1268,14 +1269,14 @@ void LiteQuery::continue_getTransactions(unsigned remaining, bool exact) { << " " << trans_lt_; td::actor::send_closure_later( manager_, &ValidatorManager::get_block_by_lt_from_db, ton::extract_addr_prefix(acc_workchain_, acc_addr_), - trans_lt_, [ Self = actor_id(this), remaining, manager = manager_ ](td::Result res) { + trans_lt_, [Self = actor_id(this), remaining, manager = manager_](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, res.move_as_error(), ton::BlockIdExt{}); } else { auto handle = res.move_as_ok(); LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, - [ Self, blkid = handle->id(), remaining ](td::Result> res) { + [Self, blkid = handle->id(), remaining](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, res.move_as_error(), blkid); @@ -1293,7 +1294,7 @@ void LiteQuery::continue_getTransactions_2(BlockIdExt blkid, Ref bloc --pending_; CHECK(!pending_); CHECK(block.not_null()); - block_ = block; + block_ = Ref(std::move(block)); blk_id_ = blkid; continue_getTransactions(remaining, true); } @@ -1339,24 +1340,122 @@ void LiteQuery::perform_getShardInfo(BlockIdExt blkid, ShardIdFull shard, bool e request_mc_block_data_state(blkid); } +void LiteQuery::load_prevKeyBlock(ton::BlockIdExt blkid, td::Promise>> promise) { + td::actor::send_closure_later(manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, + [Self = actor_id(this), blkid, promise = std::move(promise)]( + td::Result, BlockIdExt>> res) mutable { + td::actor::send_closure_later(Self, &LiteQuery::continue_loadPrevKeyBlock, blkid, + std::move(res), std::move(promise)); + }); +} + +void LiteQuery::continue_loadPrevKeyBlock(ton::BlockIdExt blkid, + td::Result, BlockIdExt>> res, + td::Promise>> promise) { + TRY_RESULT_PROMISE(promise, pair, std::move(res)); + base_blk_id_ = pair.second; + if (!base_blk_id_.is_masterchain_ext()) { + promise.set_error( + td::Status::Error(PSTRING() << "the most recent masterchain block " << base_blk_id_.to_str() << " is invalid")); + return; + } + auto state = Ref(std::move(pair.first)); + if (state.is_null()) { + promise.set_error( + td::Status::Error(PSLICE() << "obtained no valid masterchain state for block " << base_blk_id_.to_str())); + return; + } + if (blkid.seqno() > base_blk_id_.seqno()) { + promise.set_error(td::Status::Error(PSLICE() + << "client knows block " << blkid.to_str() + << " newer than the reference masterchain block " << base_blk_id_.to_str())); + return; + } + mc_state0_ = Ref(state); + if (base_blk_id_ != state->get_block_id()) { + promise.set_error(td::Status::Error(PSLICE() << "the state for " << base_blk_id_.to_str() + << " is in fact a state for different block " + << state->get_block_id().to_str())); + return; + } + if (!state->check_old_mc_block_id(blkid)) { + promise.set_error(td::Status::Error(PSLICE() << "requested masterchain block " << blkid.to_str() + << " is unknown from the perspective of reference block " + << base_blk_id_.to_str())); + return; + } + LOG(INFO) << "continuing load_prevKeyBlock(" << blkid.to_str() << ") query with a state for " + << base_blk_id_.to_str(); + auto key_blk_id = state->prev_key_block_id(blkid.seqno()); + td::actor::send_closure_later( + manager_, &ValidatorManager::get_block_data_from_db_short, key_blk_id, + [Self = actor_id(this), key_blk_id, promise = std::move(promise)](td::Result> res) mutable { + td::actor::send_closure_later(Self, &LiteQuery::finish_loadPrevKeyBlock, key_blk_id, std::move(res), + std::move(promise)); + }); +} + +void LiteQuery::finish_loadPrevKeyBlock(ton::BlockIdExt blkid, td::Result> res, + td::Promise>> promise) { + TRY_RESULT_PROMISE_PREFIX(promise, data, std::move(res), PSLICE() << "cannot load block " << blkid.to_str() << " : "); + Ref data0{std::move(data)}; + if (data0.is_null()) { + promise.set_error(td::Status::Error("no block data for key block "s + blkid.to_str())); + return; + } + promise.set_result(std::make_pair(blkid, std::move(data0))); +} + void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector param_list) { LOG(INFO) << "started a getConfigParams(" << blkid.to_str() << ", " << mode << ", ) liteserver query"; - set_continuation([ this, mode, param_list = std::move(param_list) ]() mutable { - continue_getConfigParams(mode, std::move(param_list)); - }); - request_mc_block_data_state(blkid); + if (!blkid.is_masterchain_ext()) { + fatal_error("configuration parameters can be loaded with respect to a masterchain block only"); + return; + } + if (!(mode & 0x8000)) { + // ordinary case: get configuration from masterchain state + set_continuation([this, mode, param_list = std::move(param_list)]() mutable { + continue_getConfigParams(mode, std::move(param_list)); + }); + request_mc_block_data_state(blkid); + } else { + // get configuration from previous key block + load_prevKeyBlock(blkid, [this, blkid, mode, param_list = std::move(param_list)]( + td::Result>> res) mutable { + if (res.is_error()) { + this->abort_query(res.move_as_error()); + } else { + this->base_blk_id_ = res.ok().first; + this->mc_block_ = res.move_as_ok().second; + this->continue_getConfigParams(mode, std::move(param_list)); + } + }); + } } void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) { LOG(INFO) << "completing getConfigParams(" << base_blk_id_.to_str() << ", " << mode << ", ) liteserver query"; - Ref proof1; - if (!make_mc_state_root_proof(proof1)) { + bool keyblk = (mode & 0x8000); + Ref proof1, block; + if (keyblk) { + block = mc_block_->root_cell(); + } else if (!make_mc_state_root_proof(proof1)) { return; } - vm::MerkleProofBuilder mpb{mc_state_->root_cell()}; - auto res = block::Config::extract_from_state(mpb.root(), mode); + + vm::MerkleProofBuilder mpb{keyblk ? block : mc_state_->root_cell()}; + if (keyblk) { + auto res = block::check_block_header_proof(mpb.root(), base_blk_id_); + if (res.is_error()) { + fatal_error(res.move_as_error_prefix("invalid key block header:")); + return; + } + } + + auto res = keyblk ? block::Config::extract_from_key_block(mpb.root(), mode) + : block::Config::extract_from_state(mpb.root(), mode); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -1367,9 +1466,9 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) return; } try { - if (mode & 0x2000) { + if (mode & 0x20000) { visit(cfg->get_root_cell()); - } else if (mode & 0x1000) { + } else if (mode & 0x10000) { for (int i : param_list) { visit(cfg->get_config_param(i)); } @@ -1378,7 +1477,7 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) fatal_error("error while traversing required configuration parameters: "s + err.get_msg()); return; } - auto res1 = vm::std_boc_serialize(std::move(proof1)); + auto res1 = !keyblk ? vm::std_boc_serialize(std::move(proof1)) : td::BufferSlice(); if (res1.is_error()) { fatal_error("cannot serialize Merkle proof : "s + res1.move_as_error().to_string()); return; @@ -1390,7 +1489,7 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) } LOG(INFO) << "getConfigParams() query completed"; auto b = ton::create_serialize_tl_object( - mode & 0xfff, ton::create_tl_lite_block_id(base_blk_id_), res1.move_as_ok(), res2.move_as_ok()); + mode & 0xffff, ton::create_tl_lite_block_id(base_blk_id_), res1.move_as_ok(), res2.move_as_ok()); finish_query(std::move(b)); } @@ -1495,14 +1594,14 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni LOG(INFO) << "performing a lookupBlock(" << blkid.to_str() << ", " << mode << ", " << lt << ", " << utime << ") query"; auto P = td::PromiseCreator::lambda( - [ Self = actor_id(this), manager = manager_, mode = (mode >> 4) ](td::Result res) { + [Self = actor_id(this), manager = manager_, mode = (mode >> 4)](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto handle = res.move_as_ok(); LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, - [ Self, blkid = handle->id(), mode ](td::Result> res) { + [Self, blkid = handle->id(), mode](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1650,7 +1749,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, if (mode & 0x1000) { BlockIdExt bblk = (from.seqno() > to.seqno()) ? from : to; td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, bblk, - [ Self = actor_id(this), from, to, bblk, mode ](td::Result> res) { + [Self = actor_id(this), from, to, bblk, mode](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1662,7 +1761,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, } else { td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [ Self = actor_id(this), from, to, mode ](td::Result, BlockIdExt>> res) { + [Self = actor_id(this), from, to, mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1675,7 +1774,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, } else if (mode & 2) { td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [ Self = actor_id(this), from, mode ](td::Result, BlockIdExt>> res) { + [Self = actor_id(this), from, mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1686,7 +1785,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, }); } else { td::actor::send_closure_later(manager_, &ton::validator::ValidatorManager::get_shard_client_state, false, - [ Self = actor_id(this), from, mode ](td::Result res) { + [Self = actor_id(this), from, mode](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index f1922b3b..f6353c41 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -23,6 +23,7 @@ #include "interfaces/block-handle.h" #include "interfaces/validator-manager.h" #include "interfaces/shard.h" +#include "block.hpp" #include "shard.hpp" #include "proof.hpp" @@ -45,7 +46,7 @@ class LiteQuery : public td::actor::Actor { BlockIdExt base_blk_id_, base_blk_id_alt_, blk_id_; Ref mc_state_, mc_state0_; Ref state_; - Ref mc_block_, block_; + Ref mc_block_, block_; Ref mc_proof_, mc_proof_alt_; Ref proof_link_; td::BufferSlice buffer_; @@ -131,6 +132,12 @@ class LiteQuery : public td::actor::Actor { bool adjust_last_proof_link(ton::BlockIdExt cur, Ref block_root); bool finish_proof_chain(ton::BlockIdExt id); + void load_prevKeyBlock(ton::BlockIdExt blkid, td::Promise>>); + void continue_loadPrevKeyBlock(ton::BlockIdExt blkid, td::Result, BlockIdExt>> res, + td::Promise>>); + void finish_loadPrevKeyBlock(ton::BlockIdExt blkid, td::Result> res, + td::Promise>> promise); + bool request_block_data(BlockIdExt blkid); bool request_block_state(BlockIdExt blkid); bool request_block_data_state(BlockIdExt blkid); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index b2f80d89..fc90be1d 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -4822,14 +4822,15 @@ bool ValidateQuery::check_new_state() { // seq_no:uint32 vert_seq_no:# -> checked in unpack_next_state() // gen_utime:uint32 gen_lt:uint64 -> checked in unpack_next_state() // min_ref_mc_seqno:uint32 - ton::BlockSeqno ref_mc_seqno = std::min(std::min(is_masterchain() ? id_.seqno() : mc_seqno_, min_shard_ref_mc_seqno_), - ns_.processed_upto_->min_mc_seqno()); + ton::BlockSeqno my_mc_seqno = is_masterchain() ? id_.seqno() : mc_seqno_; + ton::BlockSeqno ref_mc_seqno = + std::min(std::min(my_mc_seqno, min_shard_ref_mc_seqno_), ns_.processed_upto_->min_mc_seqno()); if (ns_.min_ref_mc_seqno_ != ref_mc_seqno) { return reject_query( PSTRING() << "new state of " << id_.to_str() << " has minimal referenced masterchain block seqno " << ns_.min_ref_mc_seqno_ << " but the value computed from all shard references and previous masterchain block reference is " - << ref_mc_seqno << " = min(" << mc_seqno_ << "," << min_shard_ref_mc_seqno_ << "," + << ref_mc_seqno << " = min(" << my_mc_seqno << "," << min_shard_ref_mc_seqno_ << "," << ns_.processed_upto_->min_mc_seqno() << ")"); } // out_msg_queue_info:^OutMsgQueueInfo diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 50fa26a6..a93fb05b 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -201,7 +201,7 @@ void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Refid(), std::move(data), handle->id(), manager_, td::Timestamp::in(10.0), std::move(P)); + run_apply_block_query(handle->id(), std::move(data), handle->id(), manager_, td::Timestamp::in(600.0), std::move(P)); } void ArchiveImporter::applied_masterchain_block(BlockHandle handle) { @@ -353,7 +353,7 @@ void ArchiveImporter::apply_shard_block_cont3(BlockHandle handle, BlockIdExt mas } TRY_RESULT_PROMISE(promise, block, create_block(handle->id(), std::move(data.second))); - run_apply_block_query(handle->id(), std::move(block), masterchain_block_id, manager_, td::Timestamp::in(10.0), + run_apply_block_query(handle->id(), std::move(block), masterchain_block_id, manager_, td::Timestamp::in(600.0), std::move(promise)); } diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 6b393b04..f49a7819 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -101,7 +101,7 @@ class Db : public td::actor::Actor { virtual void prepare_stats(td::Promise>> promise) = 0; - virtual void truncate(td::Ref state, td::Promise promise) = 0; + virtual void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) = 0; virtual void add_key_block_proof(td::Ref proof, td::Promise promise) = 0; virtual void add_key_block_proof_link(td::Ref proof_link, td::Promise promise) = 0; @@ -115,7 +115,7 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void set_async_mode(bool mode, td::Promise promise) = 0; - virtual void run_gc(UnixTime ts) = 0; + virtual void run_gc(UnixTime ts, UnixTime archive_ttl) = 0; }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index a09eaba5..1d4a7c9f 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -157,7 +157,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void update_shard_client_block_handle(BlockHandle handle, td::Promise promise) = 0; - virtual void truncate(td::Ref state, td::Promise promise) = 0; + virtual void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) = 0; virtual void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 24135198..e43e37b2 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -882,7 +882,7 @@ void ValidatorManagerImpl::send_top_shard_block_description(td::Refget_filedb_depth()); + db_ = create_db_actor(actor_id(this), db_root_); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 1520cbbb..71818e6e 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -351,7 +351,7 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } - void truncate(td::Ref state, td::Promise promise) override { + void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) override { UNREACHABLE(); } void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override { diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp new file mode 100644 index 00000000..7b6ab8e9 --- /dev/null +++ b/validator/manager-hardfork.cpp @@ -0,0 +1,563 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "manager-hardfork.hpp" +#include "validator-group.hpp" +#include "adnl/utils.hpp" +#include "downloaders/wait-block-state.hpp" +#include "downloaders/wait-block-state-merge.hpp" +#include "downloaders/wait-block-data-disk.hpp" +#include "validator-group.hpp" +#include "fabric.h" +#include "manager.h" +#include "ton/ton-io.hpp" +#include "td/utils/overloaded.h" +#include "td/utils/filesystem.h" + +namespace ton { + +namespace validator { + +void ValidatorManagerImpl::sync_complete(td::Promise promise) { + started_ = true; + + //ShardIdFull shard_id{masterchainId, shardIdAll}; + auto shard_id = shard_to_generate_; + + auto block_id = block_to_generate_; + + std::vector prev{block_id}; + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + auto v = R.move_as_ok(); + LOG(ERROR) << "created block " << v.id; + td::actor::send_closure(SelfId, &ValidatorManagerImpl::created_candidate, std::move(v)); + } else { + LOG(ERROR) << "failed to create block: " << R.move_as_error(); + std::exit(2); + } + }); + + LOG(ERROR) << "running collate query"; + run_collate_hardfork(shard_id, block_id, prev, actor_id(this), td::Timestamp::in(10.0), std::move(P)); +} + +void ValidatorManagerImpl::created_candidate(BlockCandidate candidate) { + td::write_file(db_root_ + "/static/" + candidate.id.file_hash.to_hex(), candidate.data.as_slice()).ensure(); + LOG(ERROR) << "success, block " << candidate.id << " = " << candidate.id.to_str() << " saved to disk"; + std::cout << candidate.id.to_str() << std::endl << std::flush; + std::_Exit(0); +} + +void ValidatorManagerImpl::get_block_data(BlockHandle handle, td::Promise promise) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto B = R.move_as_ok(); + promise.set_value(B->data()); + } + }); + + get_block_data_from_db(handle, std::move(P)); +} + +void ValidatorManagerImpl::get_block_proof(BlockHandle handle, td::Promise promise) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto B = R.move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db_, &Db::get_block_proof, handle, std::move(P)); +} + +void ValidatorManagerImpl::get_block_proof_link(BlockHandle handle, td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), handle, db = db_.get()](td::Result> R) mutable { + if (R.is_error()) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto B = R.move_as_ok()->export_as_proof_link().move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db, &Db::get_block_proof, handle, std::move(P)); + } else { + auto B = R.move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db_, &Db::get_block_proof_link, handle, std::move(P)); +} + +void ValidatorManagerImpl::get_key_block_proof(BlockIdExt block_id, td::Promise promise) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto B = R.move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P)); +} + +void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), block_id, db = db_.get()](td::Result> R) mutable { + if (R.is_error()) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto B = R.move_as_ok()->export_as_proof_link().move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db, &Db::get_key_block_proof, block_id, std::move(P)); + } else { + auto B = R.move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db_, &Db::get_key_block_proof_link, block_id, std::move(P)); +} + +void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { + auto R = create_ext_message(std::move(data)); + if (R.is_ok()) { + ext_messages_.emplace_back(R.move_as_ok()); + } +} + +void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { + auto R = create_ihr_message(std::move(data)); + if (R.is_ok()) { + ihr_messages_.emplace_back(R.move_as_ok()); + } +} + +void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) { + auto it = wait_state_.find(handle->id()); + if (it == wait_state_.end()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); + }); + auto id = td::actor::create_actor("waitstate", handle, 0, actor_id(this), td::Timestamp::in(10.0), + std::move(P)) + .release(); + wait_state_[handle->id()].actor_ = id; + it = wait_state_.find(handle->id()); + } + + it->second.waiting_.emplace_back( + std::pair>>(timeout, std::move(promise))); + td::actor::send_closure(it->second.actor_, &WaitBlockState::update_timeout, timeout, 0); +} + +void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, + std::move(promise)); + }); + get_block_handle(block_id, true, std::move(P)); +} + +void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) { + auto it = wait_block_data_.find(handle->id()); + if (it == wait_block_data_.end()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle->id(), std::move(R)); + }); + auto id = td::actor::create_actor("waitdata", handle, actor_id(this), td::Timestamp::in(10.0), + std::move(P)) + .release(); + wait_block_data_[handle->id()].actor_ = id; + it = wait_block_data_.find(handle->id()); + } + + it->second.waiting_.emplace_back( + std::pair>>(timeout, std::move(promise))); + td::actor::send_closure(it->second.actor_, &WaitBlockDataDisk::update_timeout, timeout); +} + +void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_data, R.move_as_ok(), 0, timeout, + std::move(promise)); + }); + get_block_handle(block_id, true, std::move(P)); +} + +void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, + td::Timestamp timeout, td::Promise> promise) { + td::actor::create_actor("merge", left_id, right_id, 0, actor_id(this), timeout, + std::move(promise)) + .release(); +} + +void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) { + CHECK(handle); + CHECK(!handle->is_zero()); + if (!handle->merge_before()) { + auto shard = handle->id().shard_full(); + auto prev_shard = handle->one_prev(true).shard_full(); + if (shard == prev_shard) { + wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(promise)); + } else { + CHECK(shard_parent(shard) == prev_shard); + bool left = shard_child(prev_shard, true) == shard; + auto P = + td::PromiseCreator::lambda([promise = std::move(promise), left](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto s = R.move_as_ok(); + auto r = s->split(); + if (r.is_error()) { + promise.set_error(r.move_as_error()); + } else { + auto v = r.move_as_ok(); + promise.set_value(left ? std::move(v.first) : std::move(v.second)); + } + } + }); + wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(P)); + } + } else { + wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), 0, timeout, std::move(promise)); + } +} + +void ValidatorManagerImpl::wait_block_proof(BlockHandle handle, td::Timestamp timeout, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::get_block_proof, handle, std::move(promise)); +} + +void ValidatorManagerImpl::wait_block_proof_short(BlockIdExt block_id, td::Timestamp timeout, + td::Promise> promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_proof, R.move_as_ok(), timeout, + std::move(promise)); + }); + get_block_handle(block_id, true, std::move(P)); +} + +void ValidatorManagerImpl::wait_block_proof_link(BlockHandle handle, td::Timestamp timeout, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::get_block_proof_link, std::move(handle), std::move(promise)); +} + +void ValidatorManagerImpl::wait_block_proof_link_short(BlockIdExt block_id, td::Timestamp timeout, + td::Promise> promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_proof_link, R.move_as_ok(), timeout, + std::move(promise)); + }); + get_block_handle(block_id, true, std::move(P)); +} + +void ValidatorManagerImpl::wait_block_signatures(BlockHandle handle, td::Timestamp timeout, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::get_block_signatures, handle, std::move(promise)); +} + +void ValidatorManagerImpl::wait_block_signatures_short(BlockIdExt block_id, td::Timestamp timeout, + td::Promise> promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_signatures, R.move_as_ok(), timeout, + std::move(promise)); + }); + get_block_handle(block_id, true, std::move(P)); +} + +void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto state = R.move_as_ok(); + promise.set_result(state->message_queue()); + } + }); + + wait_block_state(handle, 0, timeout, std::move(P)); +} + +void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, + td::Timestamp timeout, + td::Promise> promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_message_queue, R.move_as_ok(), 0, timeout, + std::move(promise)); + }); + get_block_handle(block_id, true, std::move(P)); +} + +void ValidatorManagerImpl::get_external_messages(ShardIdFull shard, + td::Promise>> promise) { + promise.set_result(ext_messages_); +} + +void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { + promise.set_result(ihr_messages_); +} + +void ValidatorManagerImpl::get_shard_blocks(BlockIdExt masterchain_block_id, + td::Promise>> promise) { +} + +void ValidatorManagerImpl::get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) { + td::actor::send_closure(db_, &Db::get_block_data, handle, std::move(promise)); +} + +void ValidatorManagerImpl::get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) { + auto P = + td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto handle = R.move_as_ok(); + td::actor::send_closure(db, &Db::get_block_data, std::move(handle), std::move(promise)); + } + }); + get_block_handle(block_id, false, std::move(P)); +} + +void ValidatorManagerImpl::get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) { + td::actor::send_closure(db_, &Db::get_block_state, handle, std::move(promise)); +} + +void ValidatorManagerImpl::get_shard_state_from_db_short(BlockIdExt block_id, + td::Promise> promise) { + auto P = + td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto handle = R.move_as_ok(); + td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(promise)); + } + }); + get_block_handle(block_id, false, std::move(P)); +} + +void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockIdExt id, + FileHash collated_data_file_hash, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); +} + +void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { + td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); +} + +void ValidatorManagerImpl::get_block_proof_from_db_short(BlockIdExt block_id, td::Promise> promise) { + auto P = + td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto handle = R.move_as_ok(); + td::actor::send_closure(db, &Db::get_block_proof, std::move(handle), std::move(promise)); + } + }); + get_block_handle(block_id, false, std::move(P)); +} + +void ValidatorManagerImpl::get_block_proof_link_from_db(ConstBlockHandle handle, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::get_block_proof_link, std::move(handle), std::move(promise)); +} + +void ValidatorManagerImpl::get_block_proof_link_from_db_short(BlockIdExt block_id, + td::Promise> promise) { + auto P = + td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto handle = R.move_as_ok(); + td::actor::send_closure(db, &Db::get_block_proof_link, std::move(handle), std::move(promise)); + } + }); + get_block_handle(block_id, false, std::move(P)); +} + +void ValidatorManagerImpl::get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_block_by_lt, account, lt, std::move(promise)); +} + +void ValidatorManagerImpl::get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_block_by_unix_time, account, ts, std::move(promise)); +} + +void ValidatorManagerImpl::get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_block_by_seqno, account, seqno, std::move(promise)); +} + +void ValidatorManagerImpl::finished_wait_state(BlockIdExt block_id, td::Result> R) { + auto it = wait_state_.find(block_id); + if (it != wait_state_.end()) { + if (R.is_error()) { + auto S = R.move_as_error(); + for (auto &X : it->second.waiting_) { + X.second.set_error(S.clone()); + } + } else { + auto r = R.move_as_ok(); + for (auto &X : it->second.waiting_) { + X.second.set_result(r); + } + } + wait_state_.erase(it); + } +} + +void ValidatorManagerImpl::finished_wait_data(BlockIdExt block_id, td::Result> R) { + auto it = wait_block_data_.find(block_id); + if (it != wait_block_data_.end()) { + if (R.is_error()) { + auto S = R.move_as_error(); + for (auto &X : it->second.waiting_) { + X.second.set_error(S.clone()); + } + } else { + auto r = R.move_as_ok(); + for (auto &X : it->second.waiting_) { + X.second.set_result(r); + } + } + wait_block_data_.erase(it); + } +} + +void ValidatorManagerImpl::get_block_handle(BlockIdExt id, bool force, td::Promise promise) { + auto it = handles_.find(id); + if (it != handles_.end()) { + auto handle = it->second.lock(); + if (handle) { + promise.set_value(std::move(handle)); + return; + } else { + handles_.erase(it); + } + } + auto P = td::PromiseCreator::lambda( + [id, force, promise = std::move(promise), SelfId = actor_id(this)](td::Result R) mutable { + BlockHandle handle; + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() == ErrorCode::notready && force) { + handle = create_empty_block_handle(id); + } else { + promise.set_error(std::move(S)); + return; + } + } else { + handle = R.move_as_ok(); + } + td::actor::send_closure(SelfId, &ValidatorManagerImpl::register_block_handle, std::move(handle), + std::move(promise)); + }); + + td::actor::send_closure(db_, &Db::get_block_handle, id, std::move(P)); +} + +void ValidatorManagerImpl::register_block_handle(BlockHandle handle, td::Promise promise) { + auto it = handles_.find(handle->id()); + if (it != handles_.end()) { + auto h = it->second.lock(); + if (h) { + promise.set_value(std::move(h)); + return; + } + handles_.erase(it); + } + handles_.emplace(handle->id(), std::weak_ptr(handle)); + promise.set_value(std::move(handle)); +} + +void ValidatorManagerImpl::start_up() { + db_ = create_db_actor(actor_id(this), db_root_); +} + +void ValidatorManagerImpl::try_get_static_file(FileHash file_hash, td::Promise promise) { + td::actor::send_closure(db_, &Db::try_get_static_file, file_hash, std::move(promise)); +} + +td::actor::ActorOwn ValidatorManagerHardforkFactory::create( + td::Ref opts, ShardIdFull shard, BlockIdExt shard_top_block_id, std::string db_root) { + return td::actor::create_actor("manager", std::move(opts), shard_top_block_id, + db_root); +} + +} // namespace validator + +} // namespace ton diff --git a/validator/manager-hardfork.h b/validator/manager-hardfork.h new file mode 100644 index 00000000..824d27f7 --- /dev/null +++ b/validator/manager-hardfork.h @@ -0,0 +1,35 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#pragma once +#include "validator/validator.h" +#include "adnl/adnl.h" + +namespace ton { + +namespace validator { + +class ValidatorManagerHardforkFactory { + public: + static td::actor::ActorOwn create(td::Ref opts, ShardIdFull shard, + BlockIdExt shard_top_block_id, std::string db_root); +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp new file mode 100644 index 00000000..506ee036 --- /dev/null +++ b/validator/manager-hardfork.hpp @@ -0,0 +1,429 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#pragma once + +#include "interfaces/validator-manager.h" +#include "interfaces/db.h" +#include "validator-group.hpp" +#include "manager-init.h" +#include "manager-hardfork.h" + +#include +#include + +namespace ton { + +namespace validator { + +class WaitBlockState; +class WaitZeroState; +class WaitShardState; +class WaitBlockDataDisk; + +class ValidatorManagerImpl : public ValidatorManager { + private: + std::vector> ext_messages_; + std::vector> ihr_messages_; + struct Compare { + bool operator()(const td::Ref &l, const td::Ref &r) const { + return l->block_id() < r->block_id(); + } + }; + std::set, Compare> shard_blocks_, out_shard_blocks_; + std::vector shard_blocks_raw_; + + struct WaitBlockStateList { + std::vector>>> waiting_; + td::actor::ActorId actor_; + }; + std::map wait_state_; + struct WaitBlockDataList { + std::vector>>> waiting_; + td::actor::ActorId actor_; + }; + std::map wait_block_data_; + + std::map> handles_; + + std::unique_ptr callback_; + td::actor::ActorOwn db_; + BlockSeqno last_masterchain_seqno_ = 0; + bool started_ = false; + td::Ref last_masterchain_state_; + //BlockHandle last_masterchain_block_; + + public: + void install_callback(std::unique_ptr new_callback, td::Promise promise) override { + callback_ = std::move(new_callback); + promise.set_value(td::Unit()); + + callback_->initial_read_complete(nullptr); + } + void add_permanent_key(PublicKeyHash key, td::Promise promise) override { + UNREACHABLE(); + } + void add_temp_key(PublicKeyHash key, td::Promise promise) override { + UNREACHABLE(); + } + void del_permanent_key(PublicKeyHash key, td::Promise promise) override { + UNREACHABLE(); + } + void del_temp_key(PublicKeyHash key, td::Promise promise) override { + UNREACHABLE(); + } + + void validate_block_is_next_proof(BlockIdExt prev_block_id, BlockIdExt next_block_id, td::BufferSlice proof, + td::Promise promise) override { + UNREACHABLE(); + } + void validate_block_proof(BlockIdExt block_id, td::BufferSlice proof, td::Promise promise) override { + UNREACHABLE(); + } + void validate_block_proof_link(BlockIdExt block_id, td::BufferSlice proof, td::Promise promise) override { + UNREACHABLE(); + } + void validate_block_proof_rel(BlockIdExt block_id, BlockIdExt rel_block_id, td::BufferSlice proof, + td::Promise promise) override { + UNREACHABLE(); + } + void validate_block(ReceivedBlock block, td::Promise promise) override { + UNREACHABLE(); + } + void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override { + UNREACHABLE(); + } + + //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; + void sync_complete(td::Promise promise) override; + void created_candidate(BlockCandidate candidate); + + void get_next_block(BlockIdExt block_id, td::Promise promise) override { + UNREACHABLE(); + } + void get_next_key_blocks(BlockIdExt block_id, td::uint32 cnt, td::Promise> promise) override { + UNREACHABLE(); + } + void get_block_data(BlockHandle handle, td::Promise promise) override; + void get_zero_state(BlockIdExt block_id, td::Promise promise) override { + UNREACHABLE(); + } + void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override { + UNREACHABLE(); + } + void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override { + UNREACHABLE(); + } + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override { + UNREACHABLE(); + } + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, + td::int64 max_length, td::Promise promise) override { + UNREACHABLE(); + } + void get_block_proof(BlockHandle handle, td::Promise promise) override; + void get_block_proof_link(BlockHandle block_id, td::Promise promise) override; + void get_key_block_proof(BlockIdExt block_id, td::Promise promise) override; + void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; + + void new_external_message(td::BufferSlice data) override; + void new_ihr_message(td::BufferSlice data) override; + void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { + UNREACHABLE(); + } + + void add_ext_server_id(adnl::AdnlNodeIdShort id) override { + UNREACHABLE(); + } + void add_ext_server_port(td::uint16 port) override { + UNREACHABLE(); + } + + void get_block_handle(BlockIdExt id, bool force, td::Promise promise) override; + + void set_block_state(BlockHandle handle, td::Ref state, + td::Promise> promise) override { + UNREACHABLE(); + } + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, + td::Promise promise) override { + UNREACHABLE(); + } + void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override { + UNREACHABLE(); + } + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override; + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override; + + void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override { + UNREACHABLE(); + } + void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, + td::Promise> promise) override; + void wait_block_data_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp, + td::Promise> promise) override; + + void set_block_proof(BlockHandle handle, td::Ref proof, td::Promise promise) override { + UNREACHABLE(); + } + void wait_block_proof(BlockHandle handle, td::Timestamp timeout, td::Promise> promise) override; + void wait_block_proof_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; + + void set_block_proof_link(BlockHandle handle, td::Ref proof, td::Promise promise) override { + UNREACHABLE(); + } + void wait_block_proof_link(BlockHandle handle, td::Timestamp timeout, + td::Promise> promise) override; + void wait_block_proof_link_short(BlockIdExt id, td::Timestamp timeout, + td::Promise> promise) override; + + void set_block_signatures(BlockHandle handle, td::Ref signatures, + td::Promise promise) override { + UNREACHABLE(); + } + void wait_block_signatures(BlockHandle handle, td::Timestamp timeout, + td::Promise> promise) override; + void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, + td::Promise> promise) override; + + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override { + promise.set_value(td::Unit()); + } + + void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override; + void wait_prev_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override; + + void wait_block_message_queue(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override; + void wait_block_message_queue_short(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override; + void get_external_messages(ShardIdFull shard, td::Promise>> promise) override; + void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; + void get_shard_blocks(BlockIdExt masterchain_block_id, + td::Promise>> promise) override; + void complete_external_messages(std::vector to_delay, + std::vector to_delete) override { + } + void complete_ihr_messages(std::vector to_delay, std::vector to_delete) override { + } + + //void set_first_block(ZeroStateIdExt state, BlockIdExt block, td::Promise promise) override; + void set_next_block(BlockIdExt prev, BlockIdExt next, td::Promise promise) override { + UNREACHABLE(); + } + + void get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) override; + void get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) override; + void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) override; + void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; + void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, + td::Promise promise) override; + void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; + void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; + void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; + void get_block_proof_link_from_db_short(BlockIdExt id, td::Promise> promise) override; + + void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, + td::Promise promise) override; + void get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts, + td::Promise promise) override; + void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, + td::Promise promise) override; + + // get block handle declared in parent class + void write_handle(BlockHandle handle, td::Promise promise) override { + UNREACHABLE(); + } + + void new_block(BlockHandle handle, td::Ref state, td::Promise promise) override { + UNREACHABLE(); + } + void new_block_cont(BlockHandle handle, td::Ref state, td::Promise promise) { + UNREACHABLE(); + } + void get_top_masterchain_state(td::Promise> promise) override { + UNREACHABLE(); + } + void get_top_masterchain_block(td::Promise promise) override { + UNREACHABLE(); + } + void get_top_masterchain_state_block(td::Promise, BlockIdExt>> promise) override { + UNREACHABLE(); + } + + void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override { + UNREACHABLE(); + } + void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override { + UNREACHABLE(); + } + void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, + td::Promise promise) override { + UNREACHABLE(); + } + void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, + td::Promise promise) override { + UNREACHABLE(); + } + void send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, + td::Promise promise) override { + UNREACHABLE(); + } + void send_get_next_key_blocks_request(BlockIdExt block_id, td::uint32 priority, + td::Promise> promise) override { + UNREACHABLE(); + } + void send_external_message(td::Ref message) override { + new_external_message(message->serialize()); + } + void send_ihr_message(td::Ref message) override { + new_ihr_message(message->serialize()); + } + void send_top_shard_block_description(td::Ref desc) override { + UNREACHABLE(); + } + void send_block_broadcast(BlockBroadcast broadcast) override { + } + + void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override { + UNREACHABLE(); + } + void get_shard_client_state(bool from_db, td::Promise promise) override { + UNREACHABLE(); + } + void subscribe_to_shard(ShardIdFull shard) override { + } + + void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { + UNREACHABLE(); + } + void get_async_serializer_state(td::Promise promise) override { + UNREACHABLE(); + } + + void try_get_static_file(FileHash file_hash, td::Promise promise) override; + + void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override { + promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); + } + + void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override { + UNREACHABLE(); + } + void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, + td::Promise promise) override { + UNREACHABLE(); + } + + void add_shard_block_description(td::Ref desc); + + void register_block_handle(BlockHandle handle, td::Promise promise); + + void finished_wait_state(BlockIdExt id, td::Result> R); + void finished_wait_data(BlockIdExt id, td::Result> R); + + void start_up() override; + + void check_is_hardfork(BlockIdExt block_id, td::Promise promise) override { + CHECK(block_id.is_masterchain()); + promise.set_result(opts_->is_hardfork(block_id)); + } + void get_vertical_seqno(BlockSeqno seqno, td::Promise promise) override { + promise.set_result(opts_->get_vertical_seqno(seqno)); + } + void run_ext_query(td::BufferSlice data, td::Promise promise) override { + UNREACHABLE(); + } + + ValidatorManagerImpl(td::Ref opts, BlockIdExt shard_to_block_id, std::string db_root) + : opts_(std::move(opts)) + , db_root_(db_root) + , shard_to_generate_(shard_to_block_id.shard_full()) + , block_to_generate_(shard_to_block_id) { + } + + public: + void update_gc_block_handle(BlockHandle handle, td::Promise promise) override { + promise.set_value(td::Unit()); + } + void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { + promise.set_result(false); + } + void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) override { + promise.set_result(false); + } + void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) override { + promise.set_result(false); + } + void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override { + promise.set_result(false); + } + void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) override { + promise.set_result(false); + } + void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { + promise.set_result(false); + } + void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { + promise.set_result(false); + } + void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) override { + promise.set_result(false); + } + void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override { + promise.set_result(false); + } + void archive(BlockHandle handle, td::Promise promise) override { + UNREACHABLE(); + } + void update_last_known_key_block(BlockHandle handle, bool send_request) override { + } + void update_shard_client_block_handle(BlockHandle handle, td::Promise promise) override { + } + + void prepare_stats(td::Promise>> promise) override { + UNREACHABLE(); + } + + void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) override { + UNREACHABLE(); + } + void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); + } + + private: + td::Ref opts_; + + private: + std::string db_root_; + ShardIdFull shard_to_generate_; + BlockIdExt block_to_generate_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 2d38c189..139911b6 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -320,6 +320,25 @@ void ValidatorManagerMasterchainStarter::got_init_block_id(BlockIdExt block_id) void ValidatorManagerMasterchainStarter::got_init_block_handle(BlockHandle handle) { handle_ = std::move(handle); + if (!handle_->received_state()) { + LOG(ERROR) << "db inconsistent: last state ( " << handle_->id() << " ) not received"; + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, 1, td::Timestamp::in(600.0), + [SelfId = actor_id(this), handle = handle_](td::Result> R) { + td::actor::send_closure( + SelfId, &ValidatorManagerMasterchainStarter::got_init_block_handle, handle); + }); + return; + } + if (!handle_->is_applied()) { + CHECK(handle_->inited_prev()); + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, handle_->one_prev(true), false, + [SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure( + SelfId, &ValidatorManagerMasterchainStarter::got_init_block_handle, R.move_as_ok()); + }); + return; + } LOG_CHECK(handle_->received_state()) << "block_id=" << handle_->id(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -401,7 +420,7 @@ void ValidatorManagerMasterchainStarter::got_gc_block_state(td::Refid().id.seqno == 0 || handle->is_key_block()); + //CHECK(handle->id().id.seqno == 0 || handle->is_key_block()); last_key_block_handle_ = std::move(handle); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -413,10 +432,6 @@ void ValidatorManagerMasterchainStarter::got_key_block_handle(BlockHandle handle } void ValidatorManagerMasterchainStarter::got_shard_block_id(BlockIdExt block_id) { - client_block_id_ = block_id; - start_shard_client(); - return; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::got_hardforks, R.move_as_ok()); @@ -437,6 +452,13 @@ void ValidatorManagerMasterchainStarter::got_hardforks(std::vector v return; } } + if (opts_->need_db_truncate()) { + auto seq = opts_->get_truncate_seqno(); + if (seq <= handle_->id().seqno()) { + got_truncate_block_seqno(seq); + return; + } + } start_shard_client(); return; } @@ -444,9 +466,10 @@ void ValidatorManagerMasterchainStarter::got_hardforks(std::vector v LOG(FATAL) << "cannot start: number of hardforks increase is too big"; return; } + has_new_hardforks_ = true; auto b = *h.rbegin(); - if (b.seqno() > handle_->id().seqno()) { + if (handle_->id().seqno() + 1 < b.seqno()) { truncated(); return; } @@ -455,8 +478,16 @@ void ValidatorManagerMasterchainStarter::got_hardforks(std::vector v return; } + got_truncate_block_seqno(b.seqno() - 1); +} + +void ValidatorManagerMasterchainStarter::got_truncate_block_seqno(BlockSeqno seqno) { BlockIdExt id; - if (state_->get_old_mc_block_id(b.seqno() - 1, id)) { + if (handle_->id().seqno() == seqno) { + got_truncate_block_handle(handle_); + return; + } + if (state_->get_old_mc_block_id(seqno, id)) { got_truncate_block_id(id); return; } @@ -465,8 +496,7 @@ void ValidatorManagerMasterchainStarter::got_hardforks(std::vector v R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::got_truncate_block_id, R.move_as_ok()->id()); }); - td::actor::send_closure(db_, &Db::get_block_by_seqno, AccountIdPrefixFull{masterchainId, 0}, b.seqno() - 1, - std::move(P)); + td::actor::send_closure(db_, &Db::get_block_by_seqno, AccountIdPrefixFull{masterchainId, 0}, seqno, std::move(P)); } void ValidatorManagerMasterchainStarter::got_truncate_block_id(BlockIdExt block_id) { @@ -491,76 +521,66 @@ void ValidatorManagerMasterchainStarter::got_truncate_block_handle(BlockHandle h void ValidatorManagerMasterchainStarter::got_truncate_state(td::Ref state) { state_ = std::move(state); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::truncated_db); + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::got_prev_key_block_handle, R.move_as_ok()); }); - td::actor::send_closure(manager_, &ValidatorManager::truncate, state_, std::move(P)); -} - -void ValidatorManagerMasterchainStarter::truncated_db() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::truncated); - }); - - auto key = state_->last_key_block_id(); - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise(std::move(P)); - - td::actor::send_closure(db_, &Db::update_init_masterchain_block, block_id_, ig.get_promise()); - - if (client_block_id_.seqno() > block_id_.seqno()) { - client_block_id_ = block_id_; - td::actor::send_closure(db_, &Db::update_shard_client_state, client_block_id_, ig.get_promise()); - } - - if (last_key_block_handle_->id().seqno() > key.seqno()) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), - promise = ig.get_promise()](td::Result R) mutable { - R.ensure(); - td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::got_prev_key_block_handle, R.move_as_ok()); - promise.set_value(td::Unit()); - }); - td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, key, false, std::move(P)); - } - - { - auto P = td::PromiseCreator::lambda( - [b = block_id_, key, db = db_, promise = ig.get_promise()](td::Result R) mutable { - if (R.is_error()) { - promise.set_value(td::Unit()); - return; - } - auto s = R.move_as_ok(); - if (s.last_block_id.seqno() <= b.seqno()) { - promise.set_value(td::Unit()); - return; - } - s.last_block_id = b; - if (s.last_written_block_id.seqno() > b.seqno()) { - s.last_written_block_id = key; - s.last_written_block_ts = 0; // may lead to extra state snapshot on disk. Does not seem like a problem - } - td::actor::send_closure(db, &Db::update_async_serializer_state, s, std::move(promise)); - }); - td::actor::send_closure(db_, &Db::get_async_serializer_state, std::move(P)); - } + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, state_->last_key_block_id(), false, + std::move(P)); } void ValidatorManagerMasterchainStarter::got_prev_key_block_handle(BlockHandle handle) { last_key_block_handle_ = std::move(handle); + //LOG_CHECK(last_key_block_handle_->inited_is_key_block() && last_key_block_handle_->is_key_block()) + // << last_key_block_handle_->id(); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::truncated); + }); + td::actor::send_closure(manager_, &ValidatorManager::truncate, block_id_.seqno(), handle_, std::move(P)); +} + +void ValidatorManagerMasterchainStarter::truncate_shard_next(BlockIdExt block_id, td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), manager = manager_, promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + auto handle = R.move_as_ok(); + handle->unsafe_clear_next(); + handle->flush(manager, handle, std::move(promise)); + }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, true, std::move(P)); } void ValidatorManagerMasterchainStarter::truncated() { - handle_->set_next(*opts_->get_hardforks().rbegin()); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + + ig.add_promise([SelfId = actor_id(this)](td::Result R) { R.ensure(); - td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::written_next); + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::truncated_next); }); - handle_->flush(manager_, handle_, std::move(P)); + + truncate_shard_next(handle_->id(), ig.get_promise()); + auto s = state_->get_shards(); + for (auto &shard : s) { + if (opts_->need_monitor(shard->shard())) { + truncate_shard_next(shard->top_block_id(), ig.get_promise()); + } + } +} + +void ValidatorManagerMasterchainStarter::truncated_next() { + if (has_new_hardforks_) { + handle_->set_next(*opts_->get_hardforks().rbegin()); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::written_next); + }); + handle_->flush(manager_, handle_, std::move(P)); + } else { + start_shard_client(); + } } void ValidatorManagerMasterchainStarter::written_next() { diff --git a/validator/manager-init.hpp b/validator/manager-init.hpp index 8b1b75ae..7dce4e47 100644 --- a/validator/manager-init.hpp +++ b/validator/manager-init.hpp @@ -98,12 +98,15 @@ class ValidatorManagerMasterchainStarter : public td::actor::Actor { void got_key_block_handle(BlockHandle handle); void got_shard_block_id(BlockIdExt block_id); void got_hardforks(std::vector hardforks); + void got_truncate_block_seqno(BlockSeqno seqno); void got_truncate_block_id(BlockIdExt block_id); void got_truncate_block_handle(BlockHandle handle); void got_truncate_state(td::Ref state); void truncated_db(); void got_prev_key_block_handle(BlockHandle handle); void truncated(); + void truncate_shard_next(BlockIdExt block_id, td::Promise promise); + void truncated_next(); void written_next(); void start_shard_client(); void finish(); @@ -117,13 +120,13 @@ class ValidatorManagerMasterchainStarter : public td::actor::Actor { BlockHandle gc_handle_; td::Ref gc_state_; BlockHandle last_key_block_handle_; + bool has_new_hardforks_{false}; td::actor::ActorId manager_; td::actor::ActorId db_; td::Promise promise_; - BlockIdExt client_block_id_; td::actor::ActorOwn client_; }; diff --git a/validator/manager.cpp b/validator/manager.cpp index 762e3be5..3d711550 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -343,7 +343,7 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro auto P = td::PromiseCreator::lambda( [promise = std::move(promise), block_id, db = db_.get()](td::Result> R) mutable { if (R.is_error()) { - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { @@ -352,7 +352,7 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro } }); - td::actor::send_closure(db, &Db::get_key_block_proof, block_id, std::move(P)); + td::actor::send_closure(db, &Db::get_key_block_proof_link, block_id, std::move(P)); } else { auto B = R.move_as_ok()->export_as_proof_link().move_as_ok(); promise.set_value(B->data()); @@ -1123,15 +1123,14 @@ void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { bool received = handle->received(); bool inited_state = handle->received_state(); - bool inited_proof = handle->inited_proof(); - bool inited_proof_link = handle->inited_proof_link(); + bool inited_proof = handle->id().is_masterchain() ? handle->inited_proof() : handle->inited_proof(); if (handle->need_flush()) { handle->flush(actor_id(this), handle, std::move(promise)); return; } - if (received && (inited_proof || inited_proof_link)) { + if (received && inited_proof) { auto it = wait_block_data_.find(handle->id()); if (it != wait_block_data_.end()) { td::actor::send_closure(it->second.actor_, &WaitBlockData::force_read_from_db); @@ -1375,7 +1374,7 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast) { } void ValidatorManagerImpl::start_up() { - db_ = create_db_actor(actor_id(this), db_root_, opts_->get_filedb_depth()); + db_ = create_db_actor(actor_id(this), db_root_); lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); token_manager_ = td::actor::create_actor("tokenmanager"); td::mkdir(db_root_ + "/tmp/").ensure(); @@ -1448,7 +1447,9 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { last_known_key_block_handle_ = last_key_block_handle_; CHECK(last_masterchain_block_handle_->is_applied()); - callback_->new_key_block(last_key_block_handle_); + if (last_known_key_block_handle_->inited_is_key_block()) { + callback_->new_key_block(last_key_block_handle_); + } gc_masterchain_handle_ = std::move(R.gc_handle); gc_masterchain_state_ = std::move(R.gc_state); @@ -1480,6 +1481,38 @@ void ValidatorManagerImpl::read_gc_list(std::vector list) { serializer_ = td::actor::create_actor("serializer", last_key_block_handle_->id(), opts_, actor_id(this)); + if (last_masterchain_block_handle_->inited_next_left()) { + auto b = last_masterchain_block_handle_->one_next(true); + if (opts_->is_hardfork(b) && !out_of_sync()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), b](td::Result R) { + if (R.is_error()) { + LOG(INFO) << "NO HARDFORK BLOCK IN STATIC FILES"; + td::actor::send_closure(SelfId, &ValidatorManagerImpl::applied_hardfork); + return; + } + + auto dataR = create_block(b, R.move_as_ok()); + dataR.ensure(); + + auto P = td::PromiseCreator::lambda([SelfId](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::applied_hardfork); + }); + run_hardfork_accept_block_query(b, dataR.move_as_ok(), SelfId, std::move(P)); + }); + td::actor::send_closure(db_, &Db::try_get_static_file, b.file_hash, std::move(P)); + return; + } + } + + if (!out_of_sync()) { + completed_prestart_sync(); + } else { + prestart_sync(); + } +} + +void ValidatorManagerImpl::applied_hardfork() { if (!out_of_sync()) { completed_prestart_sync(); } else { @@ -1488,6 +1521,13 @@ void ValidatorManagerImpl::read_gc_list(std::vector list) { } bool ValidatorManagerImpl::out_of_sync() { + auto seqno = std::min(last_masterchain_seqno_, shard_client_handle_->id().seqno()); + if (seqno < opts_->sync_upto()) { + return true; + } + if (shard_client_handle_->id().seqno() + 16 < last_masterchain_seqno_) { + return true; + } if (last_masterchain_block_handle_->unix_time() + 600 > td::Clocks::system()) { return false; } @@ -1496,7 +1536,16 @@ bool ValidatorManagerImpl::out_of_sync() { return true; } - if (validator_groups_.size() > 0 && last_known_key_block_handle_->id().seqno() <= last_masterchain_seqno_) { + bool masterchain_validator = false; + if (!validator_groups_.size()) { + auto val_set = last_masterchain_state_->get_validator_set(ShardIdFull{masterchainId}); + if (!get_validator(ShardIdFull{masterchainId}, val_set).is_zero()) { + masterchain_validator = true; + } + } + + if ((masterchain_validator || validator_groups_.size() > 0) && + last_known_key_block_handle_->id().seqno() <= last_masterchain_seqno_) { return false; } LOG(INFO) << "groups=" << validator_groups_.size() << " seqno=" << last_known_key_block_handle_->id().seqno() @@ -1538,7 +1587,7 @@ void ValidatorManagerImpl::download_next_archive() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::downloaded_archive_slice, R.move_as_ok(), true); } }); - callback_->download_archive(seqno + 1, db_root_ + "/tmp/", td::Timestamp::in(3600.0), std::move(P)); + callback_->download_archive(seqno + 1, db_root_ + "/tmp/", td::Timestamp::in(36000.0), std::move(P)); } void ValidatorManagerImpl::downloaded_archive_slice(std::string name, bool is_tmp) { @@ -2167,14 +2216,17 @@ void ValidatorManagerImpl::alarm() { try_advance_gc_masterchain_block(); alarm_timestamp() = td::Timestamp::in(1.0); if (gc_masterchain_handle_) { - td::actor::send_closure(db_, &Db::run_gc, gc_masterchain_handle_->unix_time()); + td::actor::send_closure(db_, &Db::run_gc, gc_masterchain_handle_->unix_time(), + static_cast(opts_->archive_ttl())); } if (log_status_at_.is_in_past()) { if (last_masterchain_block_handle_) { LOG(INFO) << "STATUS: last_masterchain_block_ago=" << td::format::as_time(td::Clocks::system() - last_masterchain_block_handle_->unix_time()) << " last_known_key_block_ago=" - << td::format::as_time(td::Clocks::system() - last_known_key_block_handle_->unix_time()) + << td::format::as_time(td::Clocks::system() - (last_known_key_block_handle_->inited_unix_time() + ? last_known_key_block_handle_->unix_time() + : 0)) << " shard_client_ago=" << td::format::as_time(td::Clocks::system() - (shard_client_handle_ ? shard_client_handle_->unix_time() : 0)); @@ -2268,7 +2320,7 @@ bool ValidatorManagerImpl::is_validator() { } PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { - if (!opts_->need_validate(shard)) { + if (!opts_->need_validate(shard, val_set->get_catchain_seqno())) { return PublicKeyHash::zero(); } for (auto &key : temp_keys_) { @@ -2359,8 +2411,8 @@ void ValidatorManagerImpl::prepare_stats(td::Promise state, td::Promise promise) { - td::actor::send_closure(db_, &Db::truncate, std::move(state), std::move(promise)); +void ValidatorManagerImpl::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) { + td::actor::send_closure(db_, &Db::truncate, seqno, std::move(handle), std::move(promise)); } void ValidatorManagerImpl::wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, diff --git a/validator/manager.hpp b/validator/manager.hpp index 6b5fb69c..672085d0 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -264,6 +264,7 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_block_handle(BlockHandle handle, td::Promise promise) override; bool out_of_sync(); + void applied_hardfork(); void prestart_sync(); void download_next_archive(); void downloaded_archive_slice(std::string name, bool is_tmp); @@ -512,7 +513,7 @@ class ValidatorManagerImpl : public ValidatorManager { void prepare_stats(td::Promise>> promise) override; - void truncate(td::Ref state, td::Promise promise) override; + void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) override; void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index e267e9fa..b807e3cc 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -25,10 +25,10 @@ namespace ton { namespace validator { td::Ref ValidatorManagerOptions::create( - BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, - bool allow_blockchain_init, td::ClocksBase::Duration sync_blocks_before, td::ClocksBase::Duration block_ttl, - td::ClocksBase::Duration state_ttl, td::ClocksBase::Duration archive_ttl, td::ClocksBase::Duration key_proof_ttl, - bool initial_sync_disabled) { + BlockIdExt zero_block_id, BlockIdExt init_block_id, + std::function check_shard, bool allow_blockchain_init, + td::ClocksBase::Duration sync_blocks_before, td::ClocksBase::Duration block_ttl, td::ClocksBase::Duration state_ttl, + td::ClocksBase::Duration archive_ttl, td::ClocksBase::Duration key_proof_ttl, bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, archive_ttl, key_proof_ttl, initial_sync_disabled); diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 821ee518..fdd0d108 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -33,10 +33,10 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { return init_block_id_; } bool need_monitor(ShardIdFull shard) const override { - return check_shard_(shard, ShardCheckMode::m_monitor); + return check_shard_(shard, 0, ShardCheckMode::m_monitor); } - bool need_validate(ShardIdFull shard) const override { - return check_shard_(shard, ShardCheckMode::m_validate); + bool need_validate(ShardIdFull shard, CatchainSeqno cc_seqno) const override { + return check_shard_(shard, cc_seqno, ShardCheckMode::m_validate); } bool allow_blockchain_init() const override { return allow_blockchain_init_; @@ -88,9 +88,6 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { std::vector get_hardforks() const override { return hardforks_; } - td::uint32 get_filedb_depth() const override { - return db_depth_; - } bool check_unsafe_resync_allowed(CatchainSeqno seqno) const override { return unsafe_catchains_.count(seqno) > 0; } @@ -102,6 +99,15 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { return it->second.first <= seqno ? it->second.second : 0; } } + bool need_db_truncate() const override { + return truncate_ > 0; + } + BlockSeqno get_truncate_seqno() const override { + return truncate_; + } + BlockSeqno sync_upto() const override { + return sync_upto_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -109,7 +115,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_init_block_id(BlockIdExt block_id) override { init_block_id_ = block_id; } - void set_shard_check_function(std::function check_shard) override { + void set_shard_check_function(std::function check_shard) override { check_shard_ = std::move(check_shard); } void set_allow_blockchain_init(bool value) override { @@ -136,26 +142,29 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_hardforks(std::vector vec) override { hardforks_ = std::move(vec); } - void set_filedb_depth(td::uint32 value) override { - CHECK(value <= 32); - db_depth_ = value; - } void add_unsafe_resync_catchain(CatchainSeqno seqno) override { unsafe_catchains_.insert(seqno); } void add_unsafe_catchain_rotate(BlockSeqno seqno, CatchainSeqno cc_seqno, td::uint32 value) override { unsafe_catchain_rotates_[cc_seqno] = std::make_pair(seqno, value); } + void truncate_db(BlockSeqno seqno) override { + truncate_ = seqno; + } + void set_sync_upto(BlockSeqno seqno) override { + sync_upto_ = seqno; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); } ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, - td::ClocksBase::Duration sync_blocks_before, td::ClocksBase::Duration block_ttl, - td::ClocksBase::Duration state_ttl, td::ClocksBase::Duration archive_ttl, - td::ClocksBase::Duration key_proof_ttl, bool initial_sync_disabled) + std::function check_shard, + bool allow_blockchain_init, td::ClocksBase::Duration sync_blocks_before, + td::ClocksBase::Duration block_ttl, td::ClocksBase::Duration state_ttl, + td::ClocksBase::Duration archive_ttl, td::ClocksBase::Duration key_proof_ttl, + bool initial_sync_disabled) : zero_block_id_(zero_block_id) , init_block_id_(init_block_id) , check_shard_(std::move(check_shard)) @@ -171,7 +180,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { private: BlockIdExt zero_block_id_; BlockIdExt init_block_id_; - std::function check_shard_; + std::function check_shard_; bool allow_blockchain_init_; td::ClocksBase::Duration sync_blocks_before_; td::ClocksBase::Duration block_ttl_; @@ -180,9 +189,10 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::ClocksBase::Duration key_proof_ttl_; bool initial_sync_disabled_; std::vector hardforks_; - td::uint32 db_depth_ = 2; std::set unsafe_catchains_; std::map> unsafe_catchain_rotates_; + BlockSeqno truncate_{0}; + BlockSeqno sync_upto_{0}; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index b4b93788..b825b9de 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -51,7 +51,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; virtual bool need_monitor(ShardIdFull shard) const = 0; - virtual bool need_validate(ShardIdFull shard) const = 0; + virtual bool need_validate(ShardIdFull shard, CatchainSeqno cc_seqno) const = 0; virtual bool allow_blockchain_init() const = 0; virtual td::ClocksBase::Duration sync_blocks_before() const = 0; virtual td::ClocksBase::Duration block_ttl() const = 0; @@ -64,16 +64,19 @@ struct ValidatorManagerOptions : public td::CntObject { virtual td::uint32 get_maximal_vertical_seqno() const = 0; virtual td::uint32 get_last_fork_masterchain_seqno() const = 0; virtual std::vector get_hardforks() const = 0; - virtual td::uint32 get_filedb_depth() const = 0; virtual td::uint32 key_block_utime_step() const { return 86400; } virtual bool check_unsafe_resync_allowed(CatchainSeqno seqno) const = 0; virtual td::uint32 check_unsafe_catchain_rotate(BlockSeqno seqno, CatchainSeqno cc_seqno) const = 0; + virtual bool need_db_truncate() const = 0; + virtual BlockSeqno get_truncate_seqno() const = 0; + virtual BlockSeqno sync_upto() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; - virtual void set_shard_check_function(std::function check_shard) = 0; + virtual void set_shard_check_function( + std::function check_shard) = 0; virtual void set_allow_blockchain_init(bool value) = 0; virtual void set_sync_blocks_before(td::ClocksBase::Duration value) = 0; virtual void set_block_ttl(td::ClocksBase::Duration value) = 0; @@ -82,13 +85,15 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_key_proof_ttl(td::ClocksBase::Duration value) = 0; virtual void set_initial_sync_disabled(bool value) = 0; virtual void set_hardforks(std::vector hardforks) = 0; - virtual void set_filedb_depth(td::uint32 value) = 0; virtual void add_unsafe_resync_catchain(CatchainSeqno seqno) = 0; virtual void add_unsafe_catchain_rotate(BlockSeqno seqno, CatchainSeqno cc_seqno, td::uint32 value) = 0; + virtual void truncate_db(BlockSeqno seqno) = 0; + virtual void set_sync_upto(BlockSeqno seqno) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard = [](ShardIdFull, ShardCheckMode) { return true; }, + std::function check_shard = [](ShardIdFull, CatchainSeqno, + ShardCheckMode) { return true; }, bool allow_blockchain_init = false, td::ClocksBase::Duration sync_blocks_before = 300, td::ClocksBase::Duration block_ttl = 86400 * 7, td::ClocksBase::Duration state_ttl = 3600, td::ClocksBase::Duration archive_ttl = 86400 * 365, td::ClocksBase::Duration key_proof_ttl = 86400 * 3650,