diff --git a/CMakeLists.txt b/CMakeLists.txt index 583578c6..8146bef9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -441,7 +441,7 @@ if (USE_LIBRAPTORQ) endif() add_executable(test-hello-world test/test-hello-world.cpp ) -target_link_libraries(test-hello-world tl_api) +target_link_libraries(test-hello-world tl_api crypto ton_crypto) add_executable(test-adnl test/test-adnl.cpp) target_link_libraries(test-adnl adnl adnltest dht tl_api) diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 10977b44..38f65500 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -201,6 +201,7 @@ set(BLOCK_SOURCE set(SMC_ENVELOPE_SOURCE smc-envelope/GenericAccount.cpp + smc-envelope/HighloadWallet.cpp smc-envelope/MultisigWallet.cpp smc-envelope/SmartContract.cpp smc-envelope/SmartContractCode.cpp @@ -210,6 +211,7 @@ set(SMC_ENVELOPE_SOURCE smc-envelope/WalletV3.cpp smc-envelope/GenericAccount.h + smc-envelope/HighloadWallet.h smc-envelope/MultisigWallet.h smc-envelope/SmartContract.h smc-envelope/SmartContractCode.h @@ -387,5 +389,13 @@ if (WINGETOPT_FOUND) target_link_libraries_system(dump-block wingetopt) endif() +add_executable(adjust-block block/adjust-block.cpp) +target_include_directories(adjust-block PUBLIC $ + $) +target_link_libraries(adjust-block PUBLIC ton_crypto fift-lib ton_block) +if (WINGETOPT_FOUND) + target_link_libraries_system(dump-block wingetopt) +endif() + install(TARGETS fift func RUNTIME DESTINATION bin) install(DIRECTORY fift/lib/ DESTINATION lib/fift) diff --git a/crypto/block/adjust-block.cpp b/crypto/block/adjust-block.cpp new file mode 100644 index 00000000..ba9d22cd --- /dev/null +++ b/crypto/block/adjust-block.cpp @@ -0,0 +1,203 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2019 Telegram Systems LLP +*/ +#include "block/block.h" +#include "vm/boc.h" +#include +#include "block-db.h" +#include "block-auto.h" +#include "block-parse.h" +#include "vm/cp0.h" +#include "td/utils/crypto.h" +#include + +using td::Ref; +using namespace std::literals::string_literals; + +int verbosity; + +struct IntError { + std::string err_msg; + IntError(std::string _msg) : err_msg(_msg) { + } + IntError(const char* _msg) : err_msg(_msg) { + } +}; + +int fatal(std::string str) { + std::cerr << "fatal error: " << str << std::endl; + std::exit(2); + return 2; +} + +static inline void fail_unless(td::Status res) { + if (res.is_error()) { + throw IntError{res.to_string()}; + } +} + +td::Ref load_block(std::string filename, ton::BlockIdExt& id) { + std::cerr << "loading block from bag-of-cell file " << filename << std::endl; + auto bytes_res = block::load_binary_file(filename); + if (bytes_res.is_error()) { + throw IntError{PSTRING() << "cannot load file `" << filename << "` : " << bytes_res.move_as_error()}; + } + ton::FileHash fhash; + td::sha256(bytes_res.ok(), fhash.as_slice()); + vm::BagOfCells boc; + auto res = boc.deserialize(bytes_res.move_as_ok()); + if (res.is_error()) { + throw IntError{PSTRING() << "cannot deserialize bag-of-cells " << res.move_as_error()}; + } + if (res.move_as_ok() <= 0 || boc.get_root_cell().is_null()) { + throw IntError{"cannot deserialize bag-of-cells"}; + } + auto root = boc.get_root_cell(); + std::vector prev; + ton::BlockIdExt mc_blkid; + bool after_split; + fail_unless(block::unpack_block_prev_blk_try(root, id, prev, mc_blkid, after_split, &id)); + id.file_hash = fhash; + std::cerr << "loaded block " << id.to_str() << std::endl; + return root; +} + +bool save_block(std::string filename, Ref root, ton::BlockIdExt& id) { + std::cerr << "saving block into bag-of-cell file " << filename << std::endl; + if (root.is_null()) { + throw IntError{"new block has no root"}; + } + id.root_hash = root->get_hash().bits(); + auto res = vm::std_boc_serialize(std::move(root), 31); + if (res.is_error()) { + throw IntError{PSTRING() << "cannot serialize modified block as a bag-of-cells: " + << res.move_as_error().to_string()}; + } + auto data = res.move_as_ok(); + td::sha256(data, id.file_hash.as_slice()); + auto res1 = block::save_binary_file(filename, std::move(data)); + if (res1.is_error()) { + throw IntError{PSTRING() << "cannot save file `" << filename << "` : " << res1}; + } + return true; +} + +Ref adjust_block(Ref root, int vseqno_incr, const ton::BlockIdExt& id) { + std::vector prev; + ton::BlockIdExt mc_blkid; + bool after_split; + fail_unless(block::unpack_block_prev_blk_try(root, id, prev, mc_blkid, after_split)); + std::cerr << "unpacked header of block " << id.to_str() << std::endl; + if (!id.is_masterchain()) { + throw IntError{"can modify only masterchain blocks"}; + } + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + if (!(tlb::unpack_cell(root, blk) && tlb::unpack_cell(blk.info, info))) { + throw IntError{"cannot unpack block header"}; + } + if (!info.key_block) { + throw IntError{"can modify only key blocks"}; + } + info.vert_seqno_incr = true; + info.vert_seq_no += vseqno_incr; + if (!block::tlb::t_ExtBlkRef.pack_to(info.prev_vert_ref, id, info.end_lt)) { + throw IntError{"cannot pack prev_vert_ref"}; + } + if (!(tlb::pack_cell(blk.info, info) && tlb::pack_cell(root, blk))) { + throw IntError{"cannot pack block header"}; + } + return root; +} + +void usage() { + std::cout << "usage: adjust-block [-i] \n\tor adjust-block -h\n\tAdjusts block " + "loaded from by incrementing vert_seqno by (1 by default)\n"; + std::exit(3); +} + +int main(int argc, char* const argv[]) { + int i, vseqno_incr = 1; + int new_verbosity_level = VERBOSITY_NAME(INFO); + std::string in_fname, out_fname; + while ((i = getopt(argc, argv, "hi:v:")) != -1) { + switch (i) { + case 'h': + usage(); + break; + case 'i': + vseqno_incr = td::to_integer(td::Slice(optarg)); + CHECK(vseqno_incr > 0 && vseqno_incr < 1000); + break; + case 'v': + new_verbosity_level = VERBOSITY_NAME(FATAL) + (verbosity = td::to_integer(td::Slice(optarg))); + break; + default: + usage(); + break; + } + } + SET_VERBOSITY_LEVEL(new_verbosity_level); + if (argc != optind + 2) { + usage(); + return 2; + } + in_fname = argv[optind]; + out_fname = argv[optind + 1]; + try { + ton::BlockIdExt old_id, new_id; + auto root = load_block(in_fname, old_id); + if (root.is_null()) { + return fatal("cannot load BoC from file "s + in_fname); + } + bool ok = block::gen::t_Block.validate_ref(root); + if (!ok) { + return fatal("file `"s + in_fname + " does not contain a valid block"); + } + auto adjusted = adjust_block(root, vseqno_incr, old_id); + if (adjusted.is_null()) { + return fatal("cannot adjust block"); + } + ok = block::gen::t_Block.validate_ref(root); + if (!ok) { + return fatal("modified block is not valid"); + } + new_id = old_id; + if (!save_block(out_fname, adjusted, new_id)) { + return fatal("cannot save modified block to file `"s + out_fname + "`"); + } + std::cout << "old block id: " << old_id.to_str() << std::endl; + std::cout << "new block id: " << new_id.to_str() << std::endl; + } catch (IntError& err) { + std::cerr << "internal error: " << err.err_msg << std::endl; + return 1; + } catch (vm::VmError& err) { + std::cerr << "vm error: " << err.get_msg() << std::endl; + return 1; + } + return 0; +} diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index c410e2cb..29d6ebea 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -2005,6 +2005,23 @@ bool ExtBlkRef::unpack(Ref cs_ref, ton::BlockIdExt& blkid, ton::L return true; } +bool ExtBlkRef::store(vm::CellBuilder& cb, const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const { + return cb.store_long_bool(end_lt, 64) // ext_blk_ref$_ end_lt:uint64 + && cb.store_long_bool(blkid.seqno(), 32) // seq_no:uint32 + && cb.store_bits_bool(blkid.root_hash) // root_hash:bits256 + && cb.store_bits_bool(blkid.file_hash); // file_hash:bits256 = ExtBlkRef; +} + +Ref ExtBlkRef::pack_cell(const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const { + vm::CellBuilder cb; + return store(cb, blkid, end_lt) ? cb.finalize() : Ref{}; +} + +bool ExtBlkRef::pack_to(Ref& cell, const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const { + vm::CellBuilder cb; + return store(cb, blkid, end_lt) && cb.finalize_to(cell); +} + const ExtBlkRef t_ExtBlkRef; const BlkMasterInfo t_BlkMasterInfo; diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index acbe9dd4..410a8ce5 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -921,6 +921,9 @@ struct ExtBlkRef final : TLB { } bool unpack(vm::CellSlice& cs, ton::BlockIdExt& blkid, ton::LogicalTime* end_lt = nullptr) const; bool unpack(Ref cs_ref, ton::BlockIdExt& blkid, ton::LogicalTime* end_lt = nullptr) const; + bool store(vm::CellBuilder& cb, const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const; + Ref pack_cell(const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const; + bool pack_to(Ref& cell, const ton::BlockIdExt& blkid, ton::LogicalTime end_lt) const; }; extern const ExtBlkRef t_ExtBlkRef; diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index 80723b58..bdbf08a6 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -1745,7 +1745,7 @@ td::Status unpack_block_prev_blk_ext(Ref block_root, const ton::BlockI block::gen::ExtBlkRef::Record mcref; // _ ExtBlkRef = BlkMasterInfo; ton::ShardIdFull shard; if (!(tlb::unpack_cell(block_root, blk) && tlb::unpack_cell(blk.info, info) && !info.version && - block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && !info.vert_seq_no && + block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) { return td::Status::Error("cannot unpack block header"); } @@ -1809,6 +1809,9 @@ td::Status unpack_block_prev_blk_ext(Ref block_root, const ton::BlockI } else { mc_blkid = ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, mcref.seq_no, mcref.root_hash, mcref.file_hash}; } + if (shard.is_masterchain() && info.vert_seqno_incr && !info.key_block) { + return td::Status::Error("non-key masterchain block cannot have vert_seqno_incr set"); + } return td::Status::OK(); } @@ -1817,7 +1820,7 @@ td::Status check_block_header(Ref block_root, const ton::BlockIdExt& i block::gen::BlockInfo::Record info; ton::ShardIdFull shard; if (!(tlb::unpack_cell(block_root, blk) && tlb::unpack_cell(blk.info, info) && !info.version && - block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && !info.vert_seq_no)) { + block::tlb::t_ShardIdent.unpack(info.shard.write(), shard))) { return td::Status::Error("cannot unpack block header"); } ton::BlockId hdr_id{shard, (unsigned)info.seq_no}; diff --git a/crypto/block/dump-block.cpp b/crypto/block/dump-block.cpp index ef4e1d84..a29d5c0a 100644 --- a/crypto/block/dump-block.cpp +++ b/crypto/block/dump-block.cpp @@ -191,8 +191,8 @@ void test2(vm::CellSlice& cs) { } void usage() { - std::cout << "usage: test-block [-S][]\n\tor test-block -h\n\tDumps specified blockchain block or state " - "from , or runs some tests\n\t-S\tDump a blockchain state\n"; + std::cout << "usage: dump-block [-S][]\n\tor dump-block -h\n\tDumps specified blockchain block or state " + "from , or runs some tests\n\t-S\tDump a blockchain state instead of a block\n"; std::exit(2); } diff --git a/crypto/fift/lib/Color.fif b/crypto/fift/lib/Color.fif new file mode 100644 index 00000000..bd8ed7af --- /dev/null +++ b/crypto/fift/lib/Color.fif @@ -0,0 +1,21 @@ +library Color +{ 27 emit } : esc + { char " word 27 chr swap $+ 1 ' type does create } :_ make-esc" + make-esc"[0m" ^reset + make-esc"[30m" ^black + make-esc"[31m" ^red + make-esc"[32m" ^green +make-esc"[33m" ^yellow +make-esc"[34m" ^blue +make-esc"[35m" ^magenta +make-esc"[36m" ^cyan +make-esc"[37m" ^white + // bold +make-esc"[30;1m" ^Black +make-esc"[31;1m" ^Red +make-esc"[32;1m" ^Green +make-esc"[33;1m" ^Yellow +make-esc"[34;1m" ^Blue +make-esc"[35;1m" ^Magenta +make-esc"[36;1m" ^Cyan +make-esc"[37;1m" ^White diff --git a/crypto/fift/words.cpp b/crypto/fift/words.cpp index 2c1a3083..ed703b12 100644 --- a/crypto/fift/words.cpp +++ b/crypto/fift/words.cpp @@ -96,6 +96,10 @@ void interpret_dotstack_list(IntCtx& ctx) { *ctx.output_stream << std::endl; } +void interpret_dotstack_list_dump(IntCtx& ctx) { + ctx.stack.dump(*ctx.output_stream, 3); +} + void interpret_dump(IntCtx& ctx) { ctx.stack.pop_chk().dump(*ctx.output_stream); *ctx.output_stream << ' '; @@ -105,6 +109,10 @@ void interpret_dump_internal(vm::Stack& stack) { stack.push_string(stack.pop_chk().to_string()); } +void interpret_list_dump_internal(vm::Stack& stack) { + stack.push_string(stack.pop_chk().to_lisp_string()); +} + void interpret_print_list(IntCtx& ctx) { ctx.stack.pop_chk().print_list(*ctx.output_stream); *ctx.output_stream << ' '; @@ -2434,10 +2442,12 @@ void init_words_common(Dictionary& d) { d.def_ctx_word("csr. ", interpret_dot_cellslice_rec); d.def_ctx_word(".s ", interpret_dotstack); d.def_ctx_word(".sl ", interpret_dotstack_list); + d.def_ctx_word(".sL ", interpret_dotstack_list_dump); // TMP d.def_ctx_word(".dump ", interpret_dump); d.def_ctx_word(".l ", interpret_print_list); d.def_ctx_word(".tc ", interpret_dottc); d.def_stack_word("(dump) ", interpret_dump_internal); + d.def_stack_word("(ldump) ", interpret_list_dump_internal); d.def_stack_word("(.) ", interpret_dot_internal); d.def_stack_word("(x.) ", std::bind(interpret_dothex_internal, _1, false)); d.def_stack_word("(X.) ", std::bind(interpret_dothex_internal, _1, true)); diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index dd156f24..30beb7c8 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -949,7 +949,7 @@ void define_builtins() { define_builtin_func("~touch2", TypeExpr::new_forall({X, Y}, TypeExpr::new_map(XY, TypeExpr::new_tensor({XY, Unit}))), AsmOp::Nop()); define_builtin_func("~dump", TypeExpr::new_forall({X}, TypeExpr::new_map(X, TypeExpr::new_tensor({X, Unit}))), - AsmOp::Custom("s0 DUMP", 1, 1)); + AsmOp::Custom("s0 DUMP", 1, 1), true); define_builtin_func("run_method0", TypeExpr::new_map(Int, Unit), [](auto a, auto b, auto c) { return compile_run_method(a, b, c, 0, false); }, true); define_builtin_func("run_method1", TypeExpr::new_forall({X}, TypeExpr::new_map(TypeExpr::new_tensor({Int, X}), Unit)), diff --git a/crypto/smartcont/elector-code.fc b/crypto/smartcont/elector-code.fc index fe000c14..263aa463 100644 --- a/crypto/smartcont/elector-code.fc +++ b/crypto/smartcont/elector-code.fc @@ -792,6 +792,26 @@ _ participant_list() method_id { return l; } +;; returns the list of all participants of current elections with their data +_ participant_list_extended() method_id { + var elect = get_data().begin_parse().preload_dict(); + if (elect.null?()) { + return nil; + } + var (elect_at, elect_close, min_stake, total_stake, members, failed, finished) = elect.unpack_elect(); + var l = nil; + var id = (1 << 255) + ((1 << 255) - 1); + do { + (id, var cs, var f) = members.udict_get_prev?(256, id); + if (f) { + var (stake, time, max_factor, addr, adnl_addr) = (cs~load_grams(), cs~load_uint(32), cs~load_uint(32), cs~load_uint(256), cs~load_uint(256)); + cs.end_parse(); + l = cons(pair(id, tuple4(stake, max_factor, addr, adnl_addr)), l); + } + } until (~ f); + return l; +} + ;; computes the return stake int compute_returned_stake(int wallet_addr) method_id { var cs = get_data().begin_parse(); diff --git a/crypto/smartcont/gen-zerostate.fif b/crypto/smartcont/gen-zerostate.fif index 66d68624..196ca5de 100644 --- a/crypto/smartcont/gen-zerostate.fif +++ b/crypto/smartcont/gen-zerostate.fif @@ -3,9 +3,10 @@ def? $1 { @' $1 } { "" } cond constant suffix { suffix $+ } : +suffix +256 1<<1- 15 / constant AllOnes wc_master setworkchain --17 setglobalid // negative value means a test instance of the blockchain +-239 setglobalid // negative value means a test instance of the blockchain // Initial state of Workchain 0 (Basic workchain) @@ -54,10 +55,11 @@ Libs{ x{ABACABADABACABA} s>c public_lib x{1234} x{5678} |_ s>c private_lib }Libs // libraries -GR$1700000000 // balance +GR$4999990000 // balance 0 // split_depth 0 // ticktock -2 // mode: create +AllOnes 0 * // address +6 // mode: create+setaddr register_smc dup make_special dup constant smc1_addr Masterchain over @@ -82,8 +84,8 @@ Masterchain over // code // data empty_cell // libraries -GR$1000000 // initial balance (1m test Grams) -0 0 2 register_smc +GR$1000 // initial balance (1k test Grams) +0 0 AllOnes 6 * 6 register_smc dup make_special dup constant smc2_addr Masterchain over 2dup ."free test gram giver address = " .addr cr 2dup 6 .Addr cr @@ -120,7 +122,7 @@ Libs{ x{ABACABADABACABA} s>c public_lib x{1234} x{5678} |_ s>c public_lib }Libs // libraries -0x333333333 // balance +GR$666 // balance 0 // split_depth 3 // ticktock: tick 2 // mode: create @@ -139,7 +141,8 @@ empty_cell // libraries GR$10 // balance: 10 grams 0 // split_depth 2 // ticktock: tick -2 // mode: create +AllOnes 3 * // address: -1:333...333 +6 // mode: create + setaddr register_smc dup make_special dup constant smc4_addr dup constant elector_addr Masterchain swap @@ -155,14 +158,15 @@ Masterchain swap 0 capCreateStats config.version! // max-validators max-main-validators min-validators // 9 4 1 config.validator_num! -1000 100 5 config.validator_num! +1000 100 13 config.validator_num! // min-stake max-stake min-total-stake max-factor GR$10000 GR$10000000 GR$1000000 sg~10 config.validator_stake_limits! // elected-for elect-start-before elect-end-before stakes-frozen-for // 400000 200000 4000 400000 config.election_params! -4000 2000 500 1000 config.election_params! // DEBUG +// 4000 2000 500 1000 config.election_params! // DEBUG +65536 32768 8192 32768 config.election_params! // TestNet DEBUG // config-addr = -1:5555...5555 -256 1<<1- 3 / constant config_addr +AllOnes 5 * constant config_addr config_addr config.config_smc! // elector-addr elector_addr config.elector_smc! @@ -232,8 +236,8 @@ Masterchain swap */ // pubkey amount `create-wallet1` or pubkey amount `create-wallet2` -PK'PuZPPXK5Rff9SvtoS7Y9lUuEixvy-J6aishYFj3Qn6P0pJMb GR$100000000 create-wallet1 -PK'PuYiB1zAWzr4p8j6I681+sGUrRGcn6Ylf7vXl0xaUl/w6Xfg GR$1700000000 create-wallet0 +PK'PuZPPXK5Rff9SvtoS7Y9lUuEixvy-J6aishYFj3Qn6P0pJMb GR$100 create-wallet1 +PK'PuYiB1zAWzr4p8j6I681+sGUrRGcn6Ylf7vXl0xaUl/w6Xfg GR$170 create-wallet0 /* * diff --git a/crypto/smc-envelope/HighloadWallet.cpp b/crypto/smc-envelope/HighloadWallet.cpp new file mode 100644 index 00000000..5f540780 --- /dev/null +++ b/crypto/smc-envelope/HighloadWallet.cpp @@ -0,0 +1,126 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#include "HighloadWallet.h" +#include "GenericAccount.h" +#include "SmartContractCode.h" + +#include "vm/boc.h" +#include "vm/cells/CellString.h" +#include "td/utils/base64.h" + +#include + +namespace ton { +td::Ref HighloadWallet::get_init_state(const td::Ed25519::PublicKey& public_key, + td::uint32 wallet_id) noexcept { + auto code = get_init_code(); + auto data = get_init_data(public_key, wallet_id); + return GenericAccount::get_init_state(std::move(code), std::move(data)); +} + +td::Ref HighloadWallet::get_init_message(const td::Ed25519::PrivateKey& private_key, + td::uint32 wallet_id) noexcept { + td::uint32 seqno = 0; + td::uint32 valid_until = std::numeric_limits::max(); + auto append_message = [&](auto&& cb) -> vm::CellBuilder& { + cb.store_long(wallet_id, 32).store_long(valid_until, 32).store_long(seqno, 32); + CHECK(cb.store_maybe_ref({})); + return cb; + }; + auto signature = private_key.sign(append_message(vm::CellBuilder()).finalize()->get_hash().as_slice()).move_as_ok(); + + return append_message(vm::CellBuilder().store_bytes(signature)).finalize(); +} + +td::Ref HighloadWallet::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id, + td::uint32 seqno, td::uint32 valid_until, + td::Span gifts) noexcept { + CHECK(gifts.size() <= 254); + vm::Dictionary messages(16); + for (size_t i = 0; i < gifts.size(); i++) { + auto& gift = gifts[i]; + td::int32 send_mode = 3; + auto gramms = gift.gramms; + if (gramms == -1) { + gramms = 0; + send_mode += 128; + } + vm::CellBuilder cb; + GenericAccount::store_int_message(cb, gift.destination, gramms); + cb.store_bytes("\0\0\0\0", 4); + //vm::CellString::store(cb, gift.message, 35 * 8).ensure(); + auto message_inner = cb.finalize(); + cb = {}; + cb.store_long(send_mode, 8).store_ref(message_inner); + auto key = messages.integer_key(td::make_refint(i), 16, false); + messages.set_builder(key.bits(), 16, cb); + } + + vm::CellBuilder cb; + cb.store_long(wallet_id, 32).store_long(valid_until, 32).store_long(seqno, 32); + CHECK(cb.store_maybe_ref(messages.get_root_cell())); + auto message_outer = cb.finalize(); + auto signature = private_key.sign(message_outer->get_hash().as_slice()).move_as_ok(); + return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); +} + +td::Ref HighloadWallet::get_init_code() noexcept { + return SmartContractCode::highload_wallet(); +} + +vm::CellHash HighloadWallet::get_init_code_hash() noexcept { + return get_init_code()->get_hash(); +} + +td::Ref HighloadWallet::get_init_data(const td::Ed25519::PublicKey& public_key, + td::uint32 wallet_id) noexcept { + return vm::CellBuilder() + .store_long(0, 32) + .store_long(wallet_id, 32) + .store_bytes(public_key.as_octet_string()) + .finalize(); +} + +td::Result HighloadWallet::get_seqno() const { + return TRY_VM(get_seqno_or_throw()); +} + +td::Result HighloadWallet::get_seqno_or_throw() const { + if (state_.data.is_null()) { + return 0; + } + //FIXME use get method + return static_cast(vm::load_cell_slice(state_.data).fetch_ulong(32)); +} + +td::Result HighloadWallet::get_wallet_id() const { + return TRY_VM(get_wallet_id_or_throw()); +} + +td::Result HighloadWallet::get_wallet_id_or_throw() const { + if (state_.data.is_null()) { + return 0; + } + //FIXME use get method + auto cs = vm::load_cell_slice(state_.data); + cs.skip_first(32); + return static_cast(cs.fetch_ulong(32)); +} + +} // namespace ton diff --git a/crypto/smc-envelope/HighloadWallet.h b/crypto/smc-envelope/HighloadWallet.h new file mode 100644 index 00000000..aa1e3a4b --- /dev/null +++ b/crypto/smc-envelope/HighloadWallet.h @@ -0,0 +1,54 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#pragma once + +#include "smc-envelope/SmartContract.h" +#include "vm/cells.h" +#include "Ed25519.h" +#include "block/block.h" +#include "vm/cells/CellString.h" + +namespace ton { +class HighloadWallet : ton::SmartContract { + public: + explicit HighloadWallet(State state) : ton::SmartContract(std::move(state)) { + } + static constexpr unsigned max_message_size = vm::CellString::max_bytes; + static td::Ref get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) noexcept; + static td::Ref get_init_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id) noexcept; + struct Gift { + block::StdAddress destination; + td::int64 gramms; + std::string message; + }; + static td::Ref make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id, + td::uint32 seqno, td::uint32 valid_until, td::Span gifts) noexcept; + + static td::Ref get_init_code() noexcept; + static vm::CellHash get_init_code_hash() noexcept; + static td::Ref get_init_data(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) noexcept; + + td::Result get_seqno() const; + td::Result get_wallet_id() const; + + private: + td::Result get_seqno_or_throw() const; + td::Result get_wallet_id_or_throw() const; +}; +} // namespace ton diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index ea764237..aec7bed4 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -111,7 +111,7 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref= VERBOSITY_NAME(DEBUG)) { LOG(DEBUG) << "VM log\n" << logger.res; std::ostringstream os; - res.stack->dump(os); + res.stack->dump(os, 2); LOG(DEBUG) << "VM stack:\n" << os.str(); LOG(DEBUG) << "VM exit code: " << res.code; LOG(DEBUG) << "VM accepted: " << res.accepted; diff --git a/crypto/smc-envelope/SmartContractCode.cpp b/crypto/smc-envelope/SmartContractCode.cpp index c187131d..4e17139b 100644 --- a/crypto/smc-envelope/SmartContractCode.cpp +++ b/crypto/smc-envelope/SmartContractCode.cpp @@ -39,6 +39,7 @@ const auto& get_map() { #include "smartcont/auto/simple-wallet-ext-code.cpp" #include "smartcont/auto/simple-wallet-code.cpp" #include "smartcont/auto/wallet-code.cpp" +#include "smartcont/auto/highload-wallet-code.cpp" return map; }(); return map; @@ -69,4 +70,8 @@ td::Ref SmartContractCode::simple_wallet_ext() { static auto res = load("simple-wallet-ext").move_as_ok(); return res; } +td::Ref SmartContractCode::highload_wallet() { + static auto res = load("highload-wallet").move_as_ok(); + return res; +} } // namespace ton diff --git a/crypto/smc-envelope/SmartContractCode.h b/crypto/smc-envelope/SmartContractCode.h index 059215e4..0c9e4764 100644 --- a/crypto/smc-envelope/SmartContractCode.h +++ b/crypto/smc-envelope/SmartContractCode.h @@ -26,5 +26,6 @@ class SmartContractCode { static td::Ref wallet(); static td::Ref simple_wallet(); static td::Ref simple_wallet_ext(); + static td::Ref highload_wallet(); }; } // namespace ton diff --git a/crypto/test/test-smartcont.cpp b/crypto/test/test-smartcont.cpp index c0b991b6..750ed94b 100644 --- a/crypto/test/test-smartcont.cpp +++ b/crypto/test/test-smartcont.cpp @@ -35,6 +35,7 @@ #include "smc-envelope/TestWallet.h" #include "smc-envelope/Wallet.h" #include "smc-envelope/WalletV3.h" +#include "smc-envelope/HighloadWallet.h" #include "td/utils/base64.h" #include "td/utils/crypto.h" @@ -286,6 +287,73 @@ TEST(Tonlib, WalletV3) { CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == gift_message->get_hash()); } +TEST(Tonlib, HighloadWallet) { + auto source_lookup = fift::create_mem_source_lookup(load_source("smartcont/new-highload-wallet.fif")).move_as_ok(); + source_lookup.write_file("/auto/highload-wallet-code.fif", load_source("smartcont/auto/highload-wallet-code.fif")) + .ensure(); + auto fift_output = fift::mem_run_fift(std::move(source_lookup), {"aba", "0", "239"}).move_as_ok(); + + LOG(ERROR) << fift_output.output; + auto new_wallet_pk = fift_output.source_lookup.read_file("new-wallet.pk").move_as_ok().data; + auto new_wallet_query = fift_output.source_lookup.read_file("new-wallet239-query.boc").move_as_ok().data; + auto new_wallet_addr = fift_output.source_lookup.read_file("new-wallet239.addr").move_as_ok().data; + + td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}}; + auto pub_key = priv_key.get_public_key().move_as_ok(); + auto init_state = ton::HighloadWallet::get_init_state(pub_key, 239); + auto init_message = ton::HighloadWallet::get_init_message(priv_key, 239); + auto address = ton::GenericAccount::get_address(0, init_state); + + CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32)); + + td::Ref res = ton::GenericAccount::create_ext_message(address, init_state, init_message); + + LOG(ERROR) << "---smc-envelope----"; + vm::load_cell_slice(res).print_rec(std::cerr); + LOG(ERROR) << "---fift scripts----"; + vm::load_cell_slice(vm::std_boc_deserialize(new_wallet_query).move_as_ok()).print_rec(std::cerr); + CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == res->get_hash()); + + fift_output.source_lookup.write_file("/main.fif", load_source("smartcont/highload-wallet.fif")).ensure(); + std::string order; + std::vector gifts; + auto add_order = [&](td::Slice dest_str, td::int64 gramms) { + auto g = td::to_string(gramms); + if (g.size() < 10) { + g = std::string(10 - g.size(), '0') + g; + } + order += PSTRING() << "SEND " << dest_str << " " << g.substr(0, g.size() - 9) << "." << g.substr(g.size() - 9) + << "\n"; + + ton::HighloadWallet::Gift gift; + gift.destination = block::StdAddress::parse(dest_str).move_as_ok(); + gift.gramms = gramms; + gifts.push_back(gift); + }; + std::string dest_str = "Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX"; + add_order(dest_str, 0); + add_order(dest_str, 321000000000ll); + add_order(dest_str, 321ll); + fift_output.source_lookup.write_file("/order", order).ensure(); + class ZeroOsTime : public fift::OsTime { + public: + td::uint32 now() override { + return 0; + } + }; + fift_output.source_lookup.set_os_time(std::make_unique()); + fift_output = fift::mem_run_fift(std::move(fift_output.source_lookup), {"aba", "new-wallet", "239", "123", "order"}) + .move_as_ok(); + auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data; + auto gift_message = ton::GenericAccount::create_ext_message( + address, {}, ton::HighloadWallet::make_a_gift_message(priv_key, 239, 123, 60, gifts)); + LOG(ERROR) << "---smc-envelope----"; + vm::load_cell_slice(gift_message).print_rec(std::cerr); + LOG(ERROR) << "---fift scripts----"; + vm::load_cell_slice(vm::std_boc_deserialize(wallet_query).move_as_ok()).print_rec(std::cerr); + CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == gift_message->get_hash()); +} + TEST(Tonlib, TestGiver) { auto address = block::StdAddress::parse("-1:60c04141c6a7b96d68615e7a91d265ad0f3a9a922e9ae9c901d4fa83f5d3c0d0").move_as_ok(); diff --git a/crypto/vm/continuation.cpp b/crypto/vm/continuation.cpp index d4952769..5273e159 100644 --- a/crypto/vm/continuation.cpp +++ b/crypto/vm/continuation.cpp @@ -692,7 +692,7 @@ int VmState::step() { //VM_LOG(st) << "stack:"; stack->dump(VM_LOG(st)); //VM_LOG(st) << "; cr0.refcnt = " << get_c0()->get_refcnt() - 1 << std::endl; if (stack_trace) { - stack->dump(std::cerr); + stack->dump(std::cerr, 3); } ++steps; if (code->size()) { diff --git a/crypto/vm/debugops.cpp b/crypto/vm/debugops.cpp index 911d6313..5c22740a 100644 --- a/crypto/vm/debugops.cpp +++ b/crypto/vm/debugops.cpp @@ -74,7 +74,8 @@ int exec_dump_stack(VmState* st) { d = 255; } for (int i = d; i > 0; i--) { - std::cerr << stack[i - 1].to_string() << " "; + stack[i - 1].print_list(std::cerr); + std::cerr << ' '; } std::cerr << std::endl; return 0; @@ -85,7 +86,9 @@ int exec_dump_value(VmState* st, unsigned arg) { VM_LOG(st) << "execute DUMP s" << arg; Stack& stack = st->get_stack(); if ((int)arg < stack.depth()) { - std::cerr << "#DEBUG#: s" << arg << " = " << stack[arg].to_string() << std::endl; + std::cerr << "#DEBUG#: s" << arg << " = "; + stack[arg].print_list(std::cerr); + std::cerr << std::endl; } else { std::cerr << "#DEBUG#: s" << arg << " is absent" << std::endl; } diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index 7037e7e4..0b48f90e 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -793,8 +793,7 @@ std::tuple, Ref, bool> dict_lookup_set(Ref dict, td:: std::pair, bool> pfx_dict_set(Ref dict, td::ConstBitPtr key, int m, int n, const PrefixDictionary::store_value_func_t& store_val, Dictionary::SetMode mode) { - std::cerr << "up to " << n << "-bit prefix code dictionary modification for " << m << "-bit key = " << key.to_hex(m) - << std::endl; + // std::cerr << "up to " << n << "-bit prefix code dictionary modification for " << m << "-bit key = " << key.to_hex(m) << std::endl; if (m > n) { return std::make_pair(Ref{}, false); } diff --git a/crypto/vm/stack.cpp b/crypto/vm/stack.cpp index 0b63aec9..b2ba081f 100644 --- a/crypto/vm/stack.cpp +++ b/crypto/vm/stack.cpp @@ -45,6 +45,18 @@ const char* get_exception_msg(Excno exc_no) { } } +bool StackEntry::is_list(const StackEntry* se) { + Ref tuple; + while (!se->empty()) { + tuple = se->as_tuple_range(2, 2); + if (tuple.is_null()) { + return false; + } + se = &tuple->at(1); + } + return true; +} + static const char HEX_digits[] = "0123456789ABCDEF"; std::string str_to_hex(std::string data, std::string prefix) { @@ -62,6 +74,12 @@ std::string StackEntry::to_string() const { return std::move(os).str(); } +std::string StackEntry::to_lisp_string() const { + std::ostringstream os; + print_list(os); + return std::move(os).str(); +} + void StackEntry::dump(std::ostream& os) const { switch (tp) { case t_null: @@ -130,6 +148,12 @@ void StackEntry::print_list(std::ostream& os) const { break; case t_tuple: { const auto& tuple = *static_cast>(ref); + if (is_list()) { + os << '('; + tuple[0].print_list(os); + print_list_tail(os, &tuple[1]); + break; + } auto n = tuple.size(); if (!n) { os << "[]"; @@ -137,7 +161,7 @@ void StackEntry::print_list(std::ostream& os) const { os << "["; tuple[0].print_list(os); os << "]"; - } else if (n != 2) { + } else { os << "["; unsigned c = 0; for (const auto& entry : tuple) { @@ -147,10 +171,6 @@ void StackEntry::print_list(std::ostream& os) const { entry.print_list(os); } os << ']'; - } else { - os << '('; - tuple[0].print_list(os); - tuple[1].print_list_tail(os); } break; } @@ -159,26 +179,40 @@ void StackEntry::print_list(std::ostream& os) const { } } -void StackEntry::print_list_tail(std::ostream& os) const { - switch (tp) { - case t_null: - os << ')'; - break; - case t_tuple: { - const auto& tuple = *static_cast>(ref); - if (tuple.size() == 2) { - os << ' '; - tuple[0].print_list(os); - tuple[1].print_list_tail(os); - break; - } - } - // fall through - default: +void StackEntry::print_list_tail(std::ostream& os, const StackEntry* se) { + Ref tuple; + while (!se->empty()) { + tuple = se->as_tuple_range(2, 2); + if (tuple.is_null()) { os << " . "; - print_list(os); - os << ')'; + se->print_list(os); + break; + } + os << ' '; + tuple->at(0).print_list(os); + se = &tuple->at(1); } + os << ')'; +} + +StackEntry StackEntry::make_list(std::vector&& elems) { + StackEntry tail; + std::size_t n = elems.size(); + while (n > 0) { + --n; + tail = StackEntry{vm::make_tuple_ref(std::move(elems[n]), tail)}; + } + return tail; +} + +StackEntry StackEntry::make_list(const std::vector& elems) { + StackEntry tail; + std::size_t n = elems.size(); + while (n > 0) { + --n; + tail = StackEntry{vm::make_tuple_ref(elems[n], tail)}; + } + return tail; } StackEntry::StackEntry(Ref stack_ref) : ref(std::move(stack_ref)), tp(t_stack) { @@ -611,13 +645,21 @@ Ref Stack::split_top(unsigned top_cnt, unsigned drop_cnt) { return new_stk; } -void Stack::dump(std::ostream& os, bool cr) const { +void Stack::dump(std::ostream& os, int mode) const { os << " [ "; - for (const auto& x : stack) { - os << x.to_string() << ' '; + if (mode & 2) { + for (const auto& x : stack) { + x.print_list(os); + os << ' '; + } + } else { + for (const auto& x : stack) { + x.dump(os); + os << ' '; + } } os << "] "; - if (cr) { + if (mode & 1) { os << std::endl; } } diff --git a/crypto/vm/stack.hpp b/crypto/vm/stack.hpp index 0b16f59c..1c7fb55c 100644 --- a/crypto/vm/stack.hpp +++ b/crypto/vm/stack.hpp @@ -142,6 +142,12 @@ class StackEntry { bool is(int wanted) const { return tp == wanted; } + bool is_list() const { + return is_list(this); + } + static bool is_list(const StackEntry& se) { + return is_list(&se); + } void swap(StackEntry& se) { ref.swap(se.ref); std::swap(tp, se.tp); @@ -157,8 +163,9 @@ class StackEntry { } private: + static bool is_list(const StackEntry* se); template - Ref dynamic_as() const& { + Ref dynamic_as() const & { return tp == tag ? static_cast>(ref) : td::Ref{}; } template @@ -170,7 +177,7 @@ class StackEntry { return tp == tag ? static_cast>(std::move(ref)) : td::Ref{}; } template - Ref as() const& { + Ref as() const & { return tp == tag ? Ref{td::static_cast_ref(), ref} : td::Ref{}; } template @@ -183,6 +190,8 @@ class StackEntry { } public: + static StackEntry make_list(std::vector&& elems); + static StackEntry make_list(const std::vector& elems); template static StackEntry maybe(Ref ref) { if (ref.is_null()) { @@ -191,31 +200,31 @@ class StackEntry { return ref; } } - td::RefInt256 as_int() const& { + td::RefInt256 as_int() const & { return as(); } td::RefInt256 as_int() && { return move_as(); } - Ref as_cell() const& { + Ref as_cell() const & { return as(); } Ref as_cell() && { return move_as(); } - Ref as_builder() const& { + Ref as_builder() const & { return as(); } Ref as_builder() && { return move_as(); } - Ref as_slice() const& { + Ref as_slice() const & { return as(); } Ref as_slice() && { return move_as(); } - Ref as_cont() const&; + Ref as_cont() const &; Ref as_cont() &&; Ref> as_string_ref() const { return as, t_string>(); @@ -230,16 +239,16 @@ class StackEntry { std::string as_bytes() const { return tp == t_bytes ? *as_bytes_ref() : ""; } - Ref as_box() const&; + Ref as_box() const &; Ref as_box() &&; - Ref as_tuple() const&; + Ref as_tuple() const &; Ref as_tuple() &&; - Ref as_tuple_range(unsigned max_len = 255, unsigned min_len = 0) const&; + Ref as_tuple_range(unsigned max_len = 255, unsigned min_len = 0) const &; Ref as_tuple_range(unsigned max_len = 255, unsigned min_len = 0) &&; - Ref as_atom() const&; + Ref as_atom() const &; Ref as_atom() &&; template - Ref as_object() const& { + Ref as_object() const & { return dynamic_as(); } template @@ -248,8 +257,11 @@ class StackEntry { } void dump(std::ostream& os) const; void print_list(std::ostream& os) const; - void print_list_tail(std::ostream& os) const; std::string to_string() const; + std::string to_lisp_string() const; + + private: + static void print_list_tail(std::ostream& os, const StackEntry* se); }; inline void swap(StackEntry& se1, StackEntry& se2) { @@ -490,7 +502,8 @@ class Stack : public td::CntObject { push(std::move(val)); } } - void dump(std::ostream& os, bool cr = true) const; + // mode: +1 = add eoln, +2 = Lisp-style lists + void dump(std::ostream& os, int mode = 1) const; }; } // namespace vm diff --git a/lite-client/lite-client-common.cpp b/lite-client/lite-client-common.cpp index 82503faf..184dfba3 100644 --- a/lite-client/lite-client-common.cpp +++ b/lite-client/lite-client-common.cpp @@ -9,6 +9,7 @@ using namespace std::literals::string_literals; namespace liteclient { + td::Result> deserialize_proof_chain( ton::lite_api::object_ptr f) { // deserialize proof chain @@ -75,6 +76,7 @@ td::Result> deserialize_proof_chain( LOG(DEBUG) << "deserialized a BlkProofChain of " << chain->link_count() << " links"; return std::move(chain); } + td::Ref prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, td::Ref my_addr, const block::CurrencyCollection& balance) { td::BitArray<256> rand_seed; @@ -96,4 +98,5 @@ td::Ref prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, td::Ref LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); return vm::make_tuple_ref(std::move(tuple)); } + } // namespace liteclient diff --git a/lite-client/lite-client-common.h b/lite-client/lite-client-common.h index bca2c307..54948ead 100644 --- a/lite-client/lite-client-common.h +++ b/lite-client/lite-client-common.h @@ -5,6 +5,7 @@ #include "auto/tl/lite_api.hpp" namespace liteclient { + td::Result> deserialize_proof_chain( ton::lite_api::object_ptr f); diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 54eb61b6..a4d3af51 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -497,7 +497,7 @@ void TestNode::run_init_queries() { get_server_version(0x100); } -std::string TestNode::get_word(char delim) { +td::Slice TestNode::get_word(char delim) { if (delim == ' ' || !delim) { skipspc(); } @@ -506,10 +506,33 @@ std::string TestNode::get_word(char delim) { ptr++; } std::swap(ptr, parse_ptr_); - return std::string{ptr, parse_ptr_}; + return td::Slice{ptr, parse_ptr_}; +} + +td::Slice TestNode::get_word_ext(const char* delims, const char* specials) { + if (delims[0] == ' ') { + skipspc(); + } + const char* ptr = parse_ptr_; + while (ptr < parse_end_ && !strchr(delims, *ptr)) { + if (specials && strchr(specials, *ptr)) { + if (ptr == parse_ptr_) { + ptr++; + } + break; + } + ptr++; + } + std::swap(ptr, parse_ptr_); + return td::Slice{ptr, parse_ptr_}; } bool TestNode::get_word_to(std::string& str, char delim) { + str = get_word(delim).str(); + return !str.empty(); +} + +bool TestNode::get_word_to(td::Slice& str, char delim) { str = get_word(delim); return !str.empty(); } @@ -549,12 +572,12 @@ bool TestNode::parse_account_addr(ton::WorkchainId& wc, ton::StdSmcAddress& addr return block::parse_std_account_addr(get_word(), wc, addr) || set_error("cannot parse account address"); } -bool TestNode::convert_uint64(std::string word, td::uint64& val) { +bool TestNode::convert_uint64(td::Slice word, td::uint64& val) { val = ~0ULL; if (word.empty()) { return false; } - const char* ptr = word.c_str(); + const char* ptr = word.data(); char* end = nullptr; val = std::strtoull(ptr, &end, 10); if (end == ptr + word.size()) { @@ -565,12 +588,12 @@ bool TestNode::convert_uint64(std::string word, td::uint64& val) { } } -bool TestNode::convert_int64(std::string word, td::int64& val) { +bool TestNode::convert_int64(td::Slice word, td::int64& val) { val = (~0ULL << 63); if (word.empty()) { return false; } - const char* ptr = word.c_str(); + const char* ptr = word.data(); char* end = nullptr; val = std::strtoll(ptr, &end, 10); if (end == ptr + word.size()) { @@ -581,7 +604,7 @@ bool TestNode::convert_int64(std::string word, td::int64& val) { } } -bool TestNode::convert_uint32(std::string word, td::uint32& val) { +bool TestNode::convert_uint32(td::Slice word, td::uint32& val) { td::uint64 tmp; if (convert_uint64(word, tmp) && (td::uint32)tmp == tmp) { val = (td::uint32)tmp; @@ -591,7 +614,7 @@ bool TestNode::convert_uint32(std::string word, td::uint32& val) { } } -bool TestNode::convert_int32(std::string word, td::int32& val) { +bool TestNode::convert_int32(td::Slice word, td::int32& val) { td::int64 tmp; if (convert_int64(word, tmp) && (td::int32)tmp == tmp) { val = (td::int32)tmp; @@ -631,6 +654,10 @@ int TestNode::parse_hex_digit(int c) { return -1; } +bool TestNode::parse_hash(td::Slice str, ton::Bits256& hash) { + return str.size() == 64 && parse_hash(str.data(), hash); +} + bool TestNode::parse_hash(const char* str, ton::Bits256& hash) { unsigned char* data = hash.data(); for (int i = 0; i < 32; i++) { @@ -690,15 +717,15 @@ bool TestNode::parse_block_id_ext(std::string blkid_str, ton::BlockIdExt& blkid, } bool TestNode::parse_block_id_ext(ton::BlockIdExt& blk, bool allow_incomplete) { - return parse_block_id_ext(get_word(), blk, allow_incomplete) || set_error("cannot parse BlockIdExt"); + return parse_block_id_ext(get_word().str(), blk, allow_incomplete) || set_error("cannot parse BlockIdExt"); } bool TestNode::parse_hash(ton::Bits256& hash) { auto word = get_word(); - return (!word.empty() && parse_hash(word.c_str(), hash)) || set_error("cannot parse hash"); + return parse_hash(word, hash) || set_error("cannot parse hash"); } -bool TestNode::convert_shard_id(std::string str, ton::ShardIdFull& shard) { +bool TestNode::convert_shard_id(td::Slice str, ton::ShardIdFull& shard) { shard.workchain = ton::workchainInvalid; shard.shard = 0; auto pos = str.find(':'); @@ -774,7 +801,44 @@ bool TestNode::parse_stack_value(td::Slice str, vm::StackEntry& value) { } bool TestNode::parse_stack_value(vm::StackEntry& value) { - return parse_stack_value(td::Slice{get_word()}, value) || set_error("invalid vm stack value"); + auto word = get_word_ext(" \t", "[()]"); + if (word.empty()) { + return set_error("stack value expected instead of end-of-line"); + } + if (word.size() == 1 && (word[0] == '[' || word[0] == '(')) { + int expected = (word[0] == '(' ? ')' : ']'); + std::vector values; + if (!parse_stack_values(values)) { + return false; + } + word = get_word_ext(" \t", "[()]"); + if (word.size() != 1 || word[0] != expected) { + return set_error("closing bracket expected"); + } + if (expected == ']') { + value = vm::StackEntry{std::move(values)}; + } else { + value = vm::StackEntry::make_list(std::move(values)); + } + return true; + } else { + return parse_stack_value(word, value) || set_error("invalid vm stack value"); + } +} + +bool TestNode::parse_stack_values(std::vector& values) { + values.clear(); + while (!seekeoln()) { + if (cur() == ']' || cur() == ')') { + break; + } + values.emplace_back(); + if (!parse_stack_value(values.back())) { + values.pop_back(); + return false; + } + } + return true; } bool TestNode::set_error(std::string err_msg) { @@ -865,7 +929,7 @@ bool TestNode::do_parse_line() { ton::BlockSeqno seqno{}; ton::UnixTime utime{}; unsigned count{}; - std::string word = get_word(); + std::string word = get_word().str(); skipspc(); if (word == "time") { return eoln() && get_server_time(); @@ -1027,12 +1091,11 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress bool TestNode::parse_run_method(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid, std::string method_name) { std::vector params; - while (!seekeoln()) { - vm::StackEntry param; - if (!parse_stack_value(param)) { - return false; - } - params.push_back(std::move(param)); + if (!parse_stack_values(params)) { + return set_error("cannot parse list of TVM stack values"); + } + if (!seekeoln()) { + return set_error("extra characters after a list of TVM stack values"); } if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); @@ -1278,7 +1341,7 @@ void TestNode::run_smc_method(ton::BlockIdExt ref_blk, ton::BlockIdExt blk, ton: { std::ostringstream os; os << "arguments: "; - stack->dump(os); + stack->dump(os, 3); out << os.str(); } long long gas_limit = vm::GasLimits::infty; @@ -1302,7 +1365,7 @@ void TestNode::run_smc_method(ton::BlockIdExt ref_blk, ton::BlockIdExt blk, ton: { std::ostringstream os; os << "result: "; - stack->dump(os); + stack->dump(os, 3); out << os.str(); } } diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 36830e8d..bb4f31df 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -93,6 +93,9 @@ class TestNode : public td::actor::Actor { }; void run_init_queries(); + char cur() const { + return *parse_ptr_; + } bool get_server_time(); bool get_server_version(int mode = 0); void got_server_version(td::Result res, int mode); @@ -152,8 +155,10 @@ class TestNode : public td::actor::Actor { void got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice res); bool do_parse_line(); bool show_help(std::string command); - std::string get_word(char delim = ' '); + td::Slice get_word(char delim = ' '); + td::Slice get_word_ext(const char* delims, const char* specials = nullptr); bool get_word_to(std::string& str, char delim = ' '); + bool get_word_to(td::Slice& str, char delim = ' '); int skipspc(); std::string get_line_tail(bool remove_spaces = true) const; bool eoln() const; @@ -164,11 +169,12 @@ class TestNode : public td::actor::Actor { bool parse_account_addr(ton::WorkchainId& wc, ton::StdSmcAddress& addr); static int parse_hex_digit(int c); static bool parse_hash(const char* str, ton::Bits256& hash); - static bool convert_uint64(std::string word, td::uint64& val); - static bool convert_int64(std::string word, td::int64& val); - static bool convert_uint32(std::string word, td::uint32& val); - static bool convert_int32(std::string word, td::int32& val); - static bool convert_shard_id(std::string str, ton::ShardIdFull& shard); + static bool parse_hash(td::Slice str, ton::Bits256& hash); + static bool convert_uint64(td::Slice word, td::uint64& val); + static bool convert_int64(td::Slice word, td::int64& val); + static bool convert_uint32(td::Slice word, td::uint32& val); + static bool convert_int32(td::Slice word, td::int32& val); + static bool convert_shard_id(td::Slice str, ton::ShardIdFull& shard); bool parse_hash(ton::Bits256& hash); bool parse_lt(ton::LogicalTime& lt); bool parse_uint32(td::uint32& val); @@ -177,6 +183,7 @@ class TestNode : public td::actor::Actor { bool parse_block_id_ext(std::string blk_id_string, ton::BlockIdExt& blkid, bool allow_incomplete = false) const; bool parse_stack_value(td::Slice str, vm::StackEntry& value); bool parse_stack_value(vm::StackEntry& value); + bool parse_stack_values(std::vector& values); bool register_blkid(const ton::BlockIdExt& blkid); bool show_new_blkids(bool all = false); bool complete_blkid(ton::BlockId partial_blkid, ton::BlockIdExt& complete_blkid) const; diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index ed82dbb1..8927bd05 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -56,6 +56,7 @@ set(TDUTILS_SOURCE td/utils/port/MemoryMapping.cpp td/utils/port/path.cpp td/utils/port/PollFlags.cpp + td/utils/port/rlimit.cpp td/utils/port/ServerSocketFd.cpp td/utils/port/signals.cpp td/utils/port/sleep.cpp @@ -129,6 +130,7 @@ set(TDUTILS_SOURCE td/utils/port/Poll.h td/utils/port/PollBase.h td/utils/port/PollFlags.h + td/utils/port/rlimit.h td/utils/port/RwMutex.h td/utils/port/ServerSocketFd.h td/utils/port/signals.h diff --git a/tdutils/td/utils/misc.h b/tdutils/td/utils/misc.h index d747313e..e8eea658 100644 --- a/tdutils/td/utils/misc.h +++ b/tdutils/td/utils/misc.h @@ -301,6 +301,20 @@ typename std::enable_if::value, T>::type hex_to_integer(Slic return integer_value; } +template +Result::value, T>::type> hex_to_integer_safe(Slice str) { + T integer_value = 0; + auto begin = str.begin(); + auto end = str.end(); + while (begin != end) { + if (!is_hex_digit(*begin)) { + return Status::Error("not a hex digit"); + } + integer_value = static_cast(integer_value * 16 + hex_to_int(*begin++)); + } + return integer_value; +} + double to_double(Slice str); template diff --git a/tdutils/td/utils/port/rlimit.cpp b/tdutils/td/utils/port/rlimit.cpp new file mode 100644 index 00000000..49216cc2 --- /dev/null +++ b/tdutils/td/utils/port/rlimit.cpp @@ -0,0 +1,84 @@ +#include "rlimit.h" +#if TD_LINUX +#include +#include +#include +#include +#endif + +namespace td { + +#if TD_PORT_POSIX + +namespace { + +int get_rlimit_type(RlimitType rlim_type) { + switch (rlim_type) { + case RlimitType::nofile: + return RLIMIT_NOFILE; + case RlimitType::rss: + return RLIMIT_RSS; + default: + UNREACHABLE(); + } +} + +} // namespace + +td::Status change_rlimit(RlimitType rlim_type, td::uint64 value, td::uint64 cap) { + if (cap && value > cap) { + return td::Status::Error("setrlimit(): bad argument"); + } + int resource = get_rlimit_type(rlim_type); + + struct rlimit r; + if (getrlimit(resource, &r) < 0) { + return td::Status::PosixError(errno, "failed getrlimit()"); + } + + if (cap) { + r.rlim_max = cap; + } else if (r.rlim_max < value) { + r.rlim_max = value; + } + r.rlim_cur = value; + if (setrlimit(resource, &r) < 0) { + return td::Status::PosixError(errno, "failed setrlimit()"); + } + return td::Status::OK(); +} + +td::Status change_maximize_rlimit(RlimitType rlim_type, td::uint64 value) { + int resource = get_rlimit_type(rlim_type); + + struct rlimit r; + if (getrlimit(resource, &r) < 0) { + return td::Status::PosixError(errno, "failed getrlimit()"); + } + + if (r.rlim_max < value) { + auto t = r; + t.rlim_cur = value; + t.rlim_max = value; + if (setrlimit(resource, &t) >= 0) { + return td::Status::OK(); + } + } + + r.rlim_cur = value < r.rlim_max ? value : r.rlim_max; + if (setrlimit(resource, &r) < 0) { + return td::Status::PosixError(errno, "failed setrlimit()"); + } + return td::Status::OK(); +} +#else +td::Status change_rlimit(RlimitType rlim, td::uint64 value) { + return td::Status::Error("setrlimit not implemented on WINDOWS"); +} +td::Status change_maximize_rlimit(RlimitType rlim, td::uint64 value) { + return td::Status::OK(); +} +#endif + +} // namespace td + diff --git a/tdutils/td/utils/port/rlimit.h b/tdutils/td/utils/port/rlimit.h new file mode 100644 index 00000000..b217dac8 --- /dev/null +++ b/tdutils/td/utils/port/rlimit.h @@ -0,0 +1,14 @@ +#pragma once + +#include "td/utils/port/config.h" +#include "td/utils/port/platform.h" +#include "td/utils/Status.h" + +namespace td { + +enum class RlimitType { nofile, rss }; + +td::Status change_rlimit(RlimitType rlim_type, td::uint64 value, td::uint64 cap = 0); +td::Status change_maximize_rlimit(RlimitType rlim, td::uint64 value); + +} // namespace td diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 29e26328..b86b76c3 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -7,7 +7,7 @@ Test_Fift_bug_newlize_default e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca4 Test_Fift_bug_ufits_default 51bf5a9f1ed7633a193f6fdd17a7a3af8e032dfe72a9669c85e8639aa8a7c195 Test_Fift_contfrac_default 09ebce5c91bcb70696c6fb6981d82dc3b9e3444dab608a7a1b044c0ddd778a96 Test_Fift_test_default 4e44b3382963ec89f7b5c8f2ebd85da3bc8aebad5b49f5b11b14075061477b4d -Test_Fift_test_dict_default 480d22a6ec25a232febf4eec8ff64747573f79721327e7ff3b1aa7ea4921bbb4 +Test_Fift_test_dict_default 1879f03e5fb25dcdd33f40120a6d352c246481895c9f8e45975bf3e101d9ee70 Test_Fift_test_fixed_default 278a19d56b773102caf5c1fe2997ea6c8d0d9e720eff8503feede6398a197eec Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a Test_Fift_test_sort_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a diff --git a/test/test-hello-world.cpp b/test/test-hello-world.cpp index 427528f1..3a659f57 100644 --- a/test/test-hello-world.cpp +++ b/test/test-hello-world.cpp @@ -31,6 +31,7 @@ #include "auto/tl/ton_api_json.h" #include "tl/tl_json.h" +#include "td/utils/Random.h" namespace { std::string config = R"json( @@ -135,4 +136,25 @@ int main() { return res; }; test_tl_json(ton::ton_api::make_object(create_vector_bytes())); + + td::Bits256 x; + td::Random::secure_bytes(x.as_slice()); + + auto s = x.to_hex(); + + auto v = td::hex_decode(s).move_as_ok(); + + auto w = td::buffer_to_hex(x.as_slice()); + + td::Bits256 y; + y.as_slice().copy_from(v); + + CHECK(x == y); + + auto w2 = td::hex_decode(w).move_as_ok(); + td::Bits256 z; + z.as_slice().copy_from(w2); + + LOG_CHECK(x == z) << s << " " << w; + return 0; } diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 326c82be..1fd6c091 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -303,6 +303,10 @@ class TestNode : public td::actor::Actor { void get_next_key_blocks(ton::BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override { } + void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, + + td::Promise promise) override { + } void new_key_block(ton::validator::BlockHandle handle) override { } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 38b16a84..2460201a 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -362,6 +362,9 @@ tonNode.capabilities version:int capabilities:long = tonNode.Capabilities; tonNode.success = tonNode.Success; +tonNode.archiveNotFound = tonNode.ArchiveInfo; +tonNode.archiveInfo id:long = tonNode.ArchiveInfo; + ---functions--- tonNode.getNextBlockDescription prev_block:tonNode.blockIdExt = tonNode.BlockDescription; @@ -385,6 +388,8 @@ tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; +tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; +tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; tonNode.getCapabilities = tonNode.Capabilities; @@ -417,7 +422,8 @@ db.block.info#4ac6e727 id:tonNode.blockIdExt flags:# prev_left:flags.1?tonNode.b next_right:flags.4?tonNode.blockIdExt lt:flags.13?long ts:flags.14?int - state:flags.17?int256 = db.block.Info; + state:flags.17?int256 + masterchain_ref_seqno:flags.23?int = db.block.Info; db.block.packedInfo id:tonNode.blockIdExt unixtime:int offset:long = db.block.Info; db.block.archivedInfo id:tonNode.blockIdExt flags:# next:flags.0?tonNode.blockIdExt = db.block.Info; @@ -447,6 +453,7 @@ db.state.gcBlockId block:tonNode.blockIdExt = db.state.GcBlockId; db.state.shardClient block:tonNode.blockIdExt = db.state.ShardClient; db.state.asyncSerializer block:tonNode.blockIdExt last:tonNode.blockIdExt last_ts:int = db.state.AsyncSerializer; db.state.hardforks blocks:(vector tonNode.blockIdExt) = db.state.Hardforks; +db.state.dbVersion version:int = db.state.DbVersion; db.state.key.destroyedSessions = db.state.Key; db.state.key.initBlockId = db.state.Key; @@ -454,6 +461,7 @@ db.state.key.gcBlockId = db.state.Key; db.state.key.shardClient = db.state.Key; db.state.key.asyncSerializer = db.state.Key; db.state.key.hardforks = db.state.Key; +db.state.key.dbVersion = db.state.Key; db.lt.el.key workchain:int shard:long idx:int = db.lt.Key; db.lt.desc.key workchain:int shard:long = db.lt.Key; @@ -464,12 +472,13 @@ db.lt.desc.value first_idx:int last_idx:int last_seqno:int last_lt:long last_ts: db.lt.shard.value workchain:int shard:long = db.lt.shard.Value; db.lt.status.value total_shards:int = db.lt.status.Value; -db.archive.index.key = db.archive.Key; -db.archive.package.key unixtime:int key:Bool = db.archive.Key; +db.files.index.key = db.files.Key; +db.files.package.key package_id:int key:Bool temp:Bool = db.files.Key; -db.archive.index.value packages:(vector int) key_packages:(vector int) = db.archive.index.Value; -db.archive.package.firstBlock workchain:int shard:long seqno:int lt:long = db.archive.package.FirstBlock; -db.archive.package.value unixtime:int key:Bool firstblocks:(vector db.archive.package.firstBlock) deleted:Bool = db.archive.package.Value; +db.files.index.value packages:(vector int) key_packages:(vector int) temp_packages:(vector int) = db.files.index.Value; +db.files.package.firstBlock workchain:int shard:long seqno:int unixtime:int lt:long = db.files.package.FirstBlock; +db.files.package.value package_id:int key:Bool temp:Bool firstblocks:(vector db.files.package.firstBlock) deleted:Bool + = db.files.package.Value; ---functions--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 2f8e13c9..40363811 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index e6778df5..9235ab35 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -23,6 +23,7 @@ config config:string blockchain_name:string use_callbacks_for_network:Bool ignor options config:config keystore_type:KeyStoreType = Options; options.configInfo default_wallet_id:int53 = options.ConfigInfo; +options.info config_info:options.configInfo = options.Info; key public_key:string secret:secureBytes = Key; inputKeyRegular key:key local_password:secureBytes = InputKey; @@ -81,10 +82,14 @@ query.info id:int53 valid_until:int53 body_hash:bytes = query.Info; tvm.slice bytes:string = tvm.Slice; tvm.cell bytes:string = tvm.Cell; tvm.numberDecimal number:string = tvm.Number; +tvm.tuple elements:vector = tvm.Tuple; +tvm.list elements:vector = tvm.List; tvm.stackEntrySlice slice:tvm.slice = tvm.StackEntry; tvm.stackEntryCell cell:tvm.cell = tvm.StackEntry; tvm.stackEntryNumber number:tvm.Number = tvm.StackEntry; +tvm.stackEntryTuple tuple:tvm.Tuple = tvm.StackEntry; +tvm.stackEntryList list:tvm.List = tvm.StackEntry; tvm.stackEntryUnsupported = tvm.StackEntry; smc.info id:int53 = smc.Info; @@ -121,10 +126,10 @@ liteServer.info now:int53 version:int32 capabilities:int64 = liteServer.Info; ---functions--- -init options:options = Ok; +init options:options = options.Info; close = Ok; -options.setConfig config:config = Ok; +options.setConfig config:config = options.ConfigInfo; options.validateConfig config:config = options.ConfigInfo; createNewKey local_password:secureBytes mnemonic_password:secureBytes random_extra_seed:secureBytes = Key; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index a670da5a..82702148 100644 Binary files a/tl/generate/scheme/tonlib_api.tlo and b/tl/generate/scheme/tonlib_api.tlo differ diff --git a/ton/ton-types.h b/ton/ton-types.h index fac94dc7..d4c7e33c 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -23,6 +23,7 @@ #include "td/utils/bits.h" #include "td/utils/Slice.h" #include "td/utils/UInt.h" +#include "td/utils/misc.h" namespace ton { @@ -158,6 +159,9 @@ struct BlockId { explicit operator ShardIdFull() const { return ShardIdFull{workchain, shard}; } + ShardIdFull shard_full() const { + return ShardIdFull{workchain, shard}; + } bool is_valid() const { return workchain != workchainInvalid; } @@ -274,6 +278,23 @@ struct BlockIdExt { std::string to_str() const { return id.to_str() + ':' + root_hash.to_hex() + ':' + file_hash.to_hex(); } + static td::Result from_str(td::CSlice s) { + BlockIdExt v; + char rh[65]; + char fh[65]; + auto r = sscanf(s.begin(), "(%d,%lx,%u):%64s:%64s", &v.id.workchain, &v.id.shard, &v.id.seqno, rh, fh); + if (r < 5) { + return td::Status::Error("failed to parse block id"); + } + if (strlen(rh) != 64 || strlen(fh) != 64) { + return td::Status::Error("failed to parse block id: bad roothash/filehash"); + } + TRY_RESULT(re, td::hex_decode(td::Slice(rh, 64))); + v.root_hash.as_slice().copy_from(td::Slice(re)); + TRY_RESULT(fe, td::hex_decode(td::Slice(fh, 64))); + v.file_hash.as_slice().copy_from(td::Slice(fe)); + return v; + } }; struct ZeroStateIdExt { diff --git a/tonlib/test/offline.cpp b/tonlib/test/offline.cpp index c615397e..4153e269 100644 --- a/tonlib/test/offline.cpp +++ b/tonlib/test/offline.cpp @@ -438,3 +438,94 @@ TEST(Tonlib, KeysApi) { CHECK(new_imported_key->public_key_ == key->public_key_); CHECK(new_imported_key->secret_ != key->secret_); } + +TEST(Tonlib, ConfigCache) { + using tonlib_api::make_object; + Client client; + + td::mkdir("testdir").ignore(); + // init + sync_send(client, make_object(make_object( + nullptr, make_object("testdir")))) + .ensure(); + + auto testnet = R"abc({ + "liteservers": [ + ], + "validator": { + "@type": "validator.config.global", + "zero_state": { + "workchain": -1, + "shard": -9223372036854775808, + "seqno": 0, + "root_hash": "VCSXxDHhTALFxReyTZRd8E4Ya3ySOmpOWAS4rBX9XBY=", + "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo=" + } + } +})abc"; + auto testnet2 = R"abc({ + "liteservers": [ + ], + "validator": { + "@type": "validator.config.global", + "zero_state": { + "workchain": -1, + "shard": -9223372036854775808, + "seqno": 0, + "root_hash": "VXSXxDHhTALFxReyTZRd8E4Ya3ySOmpOWAS4rBX9XBY=", + "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo=" + } + } +})abc"; + auto testnet3 = R"abc({ + "liteservers": [ + ], + "validator": { + "@type": "validator.config.global", + "zero_state": { + "workchain": -1, + "shard": -9223372036854775808, + "seqno": 0, + "root_hash": "ZXSXxDHhTALFxReyTZRd8E4Ya3ySOmpOWAS4rBX9XBY=", + "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo=" + } + } +})abc"; + auto bad = R"abc({ + "liteservers": [ + ], + "validator": { + "@type": "validator.config.global", + "zero_state": { + "workchain": -1, + "shard": -9223372036854775808, + "seqno": 0, + "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo=" + } + } +})abc"; + sync_send(client, + make_object(make_object(bad, "", true, false))) + .ensure_error(); + + sync_send(client, + make_object(make_object(testnet, "", true, false))) + .ensure(); + sync_send(client, + make_object(make_object(testnet2, "", true, false))) + .ensure(); + sync_send(client, + make_object(make_object(testnet3, "", true, false))) + .ensure(); + + sync_send(client, make_object( + make_object(testnet2, "testnet", true, false))) + .ensure_error(); + + sync_send(client, make_object( + make_object(testnet2, "testnet2", true, false))) + .ensure(); + sync_send(client, make_object( + make_object(testnet3, "testnet2", true, false))) + .ensure_error(); +} diff --git a/tonlib/test/online.cpp b/tonlib/test/online.cpp index 8eabc452..65c6f6f8 100644 --- a/tonlib/test/online.cpp +++ b/tonlib/test/online.cpp @@ -533,14 +533,12 @@ int main(int argc, char* argv[]) { Client client; { - auto info = sync_send(client, make_object( - make_object(global_config_str, "", false, false))) + auto info = sync_send(client, make_object(make_object( + make_object(global_config_str, "", false, false), + make_object(keystore_dir)))) .move_as_ok(); - default_wallet_id = static_cast(info->default_wallet_id_); - sync_send(client, make_object(make_object( - make_object(global_config_str, "", false, false), - make_object(keystore_dir)))) - .ensure(); + default_wallet_id = static_cast(info->config_info_->default_wallet_id_); + LOG(ERROR) << default_wallet_id; } // wait till client is synchronized with blockchain. diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 95a089d7..a2639210 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -66,6 +66,16 @@ struct SendMessage { }; } // namespace int_api +template +R downcast_call2(O&& o, F&& f, R res = {}) { + downcast_call(o, [&](auto& x) { res = f(x); }); + return res; +} + +tonlib_api::object_ptr to_tonlib_api(const TonlibClient::FullConfig& full_config) { + return tonlib_api::make_object(full_config.wallet_id); +} + class TonlibQueryActor : public td::actor::Actor { public: TonlibQueryActor(td::actor::ActorShared client) : client_(std::move(client)) { @@ -867,7 +877,7 @@ void TonlibClient::update_last_block_state(LastBlockState state, td::uint32 conf return; } - last_block_storage_.save_state(blockchain_name_, state); + last_block_storage_.save_state(last_state_key_, state); } void TonlibClient::update_sync_state(LastBlockSyncState state, td::uint32 config_generation) { @@ -889,7 +899,7 @@ void TonlibClient::update_sync_state(LastBlockSyncState state, td::uint32 config } } -void TonlibClient::init_last_block(td::optional o_master_config) { +void TonlibClient::init_last_block(LastBlockState state) { ref_cnt_++; class Callback : public LastBlock::Callback { public: @@ -907,31 +917,8 @@ void TonlibClient::init_last_block(td::optional o_master_config) { td::actor::ActorShared client_; td::uint32 config_generation_; }; - LastBlockState state; - td::Result r_state; - if (!ignore_cache_) { - r_state = last_block_storage_.get_state(blockchain_name_); - } - if (ignore_cache_ || r_state.is_error()) { - LOG_IF(WARNING, !ignore_cache_) << "Unknown LastBlockState: " << r_state.error(); - state.zero_state_id = ton::ZeroStateIdExt(config_.zero_state_id.id.workchain, config_.zero_state_id.root_hash, - config_.zero_state_id.file_hash), - state.last_block_id = config_.zero_state_id; - state.last_key_block_id = config_.zero_state_id; - last_block_storage_.save_state(blockchain_name_, state); - } else { - state = r_state.move_as_ok(); - } - - if (o_master_config) { - auto master_config = o_master_config.unwrap(); - if (master_config.init_block_id.is_valid() && - state.last_key_block_id.id.seqno < master_config.init_block_id.id.seqno) { - state.last_key_block_id = master_config.init_block_id; - LOG(INFO) << "Use init block from MASTER config: " << master_config.init_block_id.to_str(); - } - } + last_block_storage_.save_state(last_state_key_, state); raw_last_block_ = td::actor::create_actor( td::actor::ActorOptions().with_name("LastBlock").with_poll(false), get_client_ref(), std::move(state), config_, @@ -1013,8 +1000,8 @@ tonlib_api::object_ptr TonlibClient::static_request( return tonlib_api::make_object(400, "Request is empty"); } - tonlib_api::object_ptr response; - downcast_call(*function, [&response](auto& request) { response = TonlibClient::do_static_request(request); }); + auto response = downcast_call2>( + *function, [](auto& request) { return TonlibClient::do_static_request(request); }); VLOG(tonlib_query) << " answer static query " << to_string(response); return response; } @@ -1029,7 +1016,6 @@ bool TonlibClient::is_static_request(td::int32 id) { case tonlib_api::testGiver_getAccountAddress::ID: case tonlib_api::packAccountAddress::ID: case tonlib_api::unpackAccountAddress::ID: - case tonlib_api::options_validateConfig::ID: case tonlib_api::getBip39Hints::ID: case tonlib_api::setLogStream::ID: case tonlib_api::getLogStream::ID: @@ -1190,7 +1176,7 @@ tonlib_api::object_ptr TonlibClient::do_static_request(tonli } td::Status TonlibClient::do_request(const tonlib_api::init& request, - td::Promise>&& promise) { + td::Promise>&& promise) { if (state_ != State::Uninited) { return td::Status::Error(400, "Tonlib is already inited"); } @@ -1201,23 +1187,24 @@ td::Status TonlibClient::do_request(const tonlib_api::init& request, return TonlibError::EmptyField("options.keystore_type"); } - td::Result> r_kv; - downcast_call( + auto r_kv = downcast_call2>>( *request.options_->keystore_type_, td::overloaded( - [&](tonlib_api::keyStoreTypeDirectory& directory) { r_kv = KeyValue::create_dir(directory.directory_); }, - [&](tonlib_api::keyStoreTypeInMemory& inmemory) { r_kv = KeyValue::create_inmemory(); })); + [](tonlib_api::keyStoreTypeDirectory& directory) { return KeyValue::create_dir(directory.directory_); }, + [](tonlib_api::keyStoreTypeInMemory& inmemory) { return KeyValue::create_inmemory(); })); TRY_RESULT(kv, std::move(r_kv)); kv_ = std::shared_ptr(kv.release()); key_storage_.set_key_value(kv_); last_block_storage_.set_key_value(kv_); + auto res = tonlib_api::make_object(); if (request.options_->config_) { TRY_RESULT(full_config, validate_config(std::move(request.options_->config_))); + res->config_info_ = to_tonlib_api(full_config); set_config(std::move(full_config)); } state_ = State::Running; - promise.set_value(tonlib_api::make_object()); + promise.set_value(std::move(res)); return td::Status::OK(); } @@ -1291,21 +1278,55 @@ td::Result TonlibClient::validate_config(tonlib_api::o } td::optional o_master_config; + std::string last_state_key; if (config->blockchain_name_.empty()) { + last_state_key = new_config.zero_state_id.root_hash.as_slice().str(); o_master_config = get_default_master_config().by_root_hash(new_config.zero_state_id.root_hash); } else { + last_state_key = config->blockchain_name_; o_master_config = get_default_master_config().by_name(config->blockchain_name_); } if (o_master_config && o_master_config.value().zero_state_id != new_config.zero_state_id) { return TonlibError::InvalidConfig("zero_state differs from embedded zero_state"); } + + LastBlockState state; + td::Result r_state; + if (!config->ignore_cache_) { + r_state = last_block_storage_.get_state(last_state_key); + } + auto zero_state = ton::ZeroStateIdExt(new_config.zero_state_id.id.workchain, new_config.zero_state_id.root_hash, + new_config.zero_state_id.file_hash); + if (config->ignore_cache_ || r_state.is_error()) { + LOG_IF(WARNING, !config->ignore_cache_) << "Unknown LastBlockState: " << r_state.error(); + state.zero_state_id = zero_state; + state.last_block_id = new_config.zero_state_id; + state.last_key_block_id = new_config.zero_state_id; + } else { + state = r_state.move_as_ok(); + if (state.zero_state_id != zero_state) { + LOG(ERROR) << state.zero_state_id.to_str() << " " << zero_state.to_str(); + return TonlibError::InvalidConfig("zero_state differs from cached zero_state"); + } + } + + if (o_master_config) { + auto master_config = o_master_config.unwrap(); + if (master_config.init_block_id.is_valid() && + state.last_key_block_id.id.seqno < master_config.init_block_id.id.seqno) { + state.last_key_block_id = master_config.init_block_id; + LOG(INFO) << "Use init block from MASTER config: " << master_config.init_block_id.to_str(); + } + } + FullConfig res; res.config = std::move(new_config); - res.o_master_config = std::move(o_master_config); - res.ignore_cache = config->ignore_cache_; res.use_callbacks_for_network = config->use_callbacks_for_network_; res.wallet_id = td::as(res.config.zero_state_id.root_hash.as_slice().data()); + res.last_state_key = std::move(last_state_key); + res.last_state = std::move(state); + return std::move(res); } @@ -1313,12 +1334,11 @@ void TonlibClient::set_config(FullConfig full_config) { config_ = std::move(full_config.config); config_generation_++; wallet_id_ = full_config.wallet_id; - blockchain_name_ = config_.zero_state_id.root_hash.as_slice().str(); + last_state_key_ = full_config.last_state_key; use_callbacks_for_network_ = full_config.use_callbacks_for_network; - ignore_cache_ = full_config.ignore_cache; init_ext_client(); - init_last_block(std::move(full_config.o_master_config)); + init_last_block(std::move(full_config.last_state)); init_last_config(); client_.set_client(get_client_ref()); } @@ -1332,23 +1352,23 @@ td::Status TonlibClient::do_request(const tonlib_api::close& request, return td::Status::OK(); } -tonlib_api::object_ptr TonlibClient::do_static_request( - tonlib_api::options_validateConfig& request) { - auto r_config = validate_config(std::move(request.config_)); - if (r_config.is_error()) { - return status_to_tonlib_api(r_config.move_as_error()); - } - return tonlib_api::make_object(r_config.ok().wallet_id); +td::Status TonlibClient::do_request(tonlib_api::options_validateConfig& request, + td::Promise>&& promise) { + TRY_RESULT(config, validate_config(std::move(request.config_))); + auto res = to_tonlib_api(config); + promise.set_value(std::move(res)); + return td::Status::OK(); } td::Status TonlibClient::do_request(tonlib_api::options_setConfig& request, - td::Promise>&& promise) { + td::Promise>&& promise) { if (!request.config_) { return TonlibError::EmptyField("config"); } TRY_RESULT(config, validate_config(std::move(request.config_))); + auto res = to_tonlib_api(config); set_config(std::move(config)); - promise.set_value(tonlib_api::make_object()); + promise.set_value(std::move(res)); return td::Status::OK(); } @@ -1603,11 +1623,9 @@ td::Result from_tonlib(tonlib_api::inputKeyRegular& input_ } td::Result from_tonlib(tonlib_api::InputKey& input_key) { - td::Result r_key; - tonlib_api::downcast_call( - input_key, td::overloaded([&](tonlib_api::inputKeyRegular& input_key) { r_key = from_tonlib(input_key); }, - [&](tonlib_api::inputKeyFake&) { r_key = KeyStorage::fake_input_key(); })); - return r_key; + return downcast_call2>( + input_key, td::overloaded([&](tonlib_api::inputKeyRegular& input_key) { return from_tonlib(input_key); }, + [&](tonlib_api::inputKeyFake&) { return KeyStorage::fake_input_key(); })); } // ton::TestWallet @@ -2195,6 +2213,98 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getState& request, return td::Status::OK(); } +bool is_list(vm::StackEntry entry) { + while (true) { + if (entry.type() == vm::StackEntry::Type::t_null) { + return true; + } + if (entry.type() != vm::StackEntry::Type::t_tuple) { + return false; + } + if (entry.as_tuple()->size() != 2) { + return false; + } + entry = entry.as_tuple()->at(1); + } +}; +auto to_tonlib_api(const vm::StackEntry& entry) -> tonlib_api::object_ptr { + switch (entry.type()) { + case vm::StackEntry::Type::t_int: + return tonlib_api::make_object( + tonlib_api::make_object(dec_string(entry.as_int()))); + case vm::StackEntry::Type::t_slice: + return tonlib_api::make_object(tonlib_api::make_object( + to_bytes(vm::CellBuilder().append_cellslice(entry.as_slice()).finalize()))); + case vm::StackEntry::Type::t_cell: + return tonlib_api::make_object( + tonlib_api::make_object(to_bytes(entry.as_cell()))); + case vm::StackEntry::Type::t_null: + case vm::StackEntry::Type::t_tuple: { + std::vector> elements; + if (is_list(entry)) { + auto node = entry; + while (node.type() == vm::StackEntry::Type::t_tuple) { + elements.push_back(to_tonlib_api(node.as_tuple()->at(0))); + node = node.as_tuple()->at(1); + } + return tonlib_api::make_object( + tonlib_api::make_object(std::move(elements))); + + } else { + for (auto& element : *entry.as_tuple()) { + elements.push_back(to_tonlib_api(element)); + } + return tonlib_api::make_object( + tonlib_api::make_object(std::move(elements))); + } + } + + default: + return tonlib_api::make_object(); + } +}; + +td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { + // TODO: error codes + // downcast_call + return downcast_call2>( + entry, + td::overloaded( + [&](tonlib_api::tvm_stackEntryUnsupported& cell) { return td::Status::Error("Unsuppored stack entry"); }, + [&](tonlib_api::tvm_stackEntrySlice& cell) -> td::Result { + TRY_RESULT(res, vm::std_boc_deserialize(cell.slice_->bytes_)); + return vm::StackEntry{std::move(res)}; + }, + [&](tonlib_api::tvm_stackEntryCell& cell) -> td::Result { + TRY_RESULT(res, vm::std_boc_deserialize(cell.cell_->bytes_)); + return vm::StackEntry{std::move(res)}; + }, + [&](tonlib_api::tvm_stackEntryTuple& tuple) -> td::Result { + std::vector elements; + for (auto& element : tuple.tuple_->elements_) { + TRY_RESULT(new_element, from_tonlib_api(*element)); + elements.push_back(std::move(new_element)); + } + return td::Ref(true, std::move(elements)); + }, + [&](tonlib_api::tvm_stackEntryList& tuple) -> td::Result { + vm::StackEntry tail; + for (auto& element : td::reversed(tuple.list_->elements_)) { + TRY_RESULT(new_element, from_tonlib_api(*element)); + tail = vm::make_tuple_ref(std::move(new_element), std::move(tail)); + } + return tail; + }, + [&](tonlib_api::tvm_stackEntryNumber& number) -> td::Result { + auto& dec = *number.number_; + auto num = td::dec_string_to_int256(dec.number_); + if (num.is_null()) { + return td::Status::Error("Failed to parse dec string to int256"); + } + return num; + })); +} + td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, td::Promise>&& promise) { auto it = smcs_.find(request.id_); @@ -2209,65 +2319,17 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, [&](tonlib_api::smc_methodIdName& name) { args.set_method_id(name.name_); })); td::Ref stack(true); td::Status status; - // TODO: error codes - // downcast_call for (auto& entry : request.stack_) { - downcast_call(*entry, td::overloaded( - [&](tonlib_api::tvm_stackEntryUnsupported& cell) { - status = td::Status::Error("Unsuppored stack entry"); - }, - [&](tonlib_api::tvm_stackEntrySlice& cell) { - auto r_cell = vm::std_boc_deserialize(cell.slice_->bytes_); - if (r_cell.is_error()) { - status = r_cell.move_as_error(); - return; - } - stack.write().push_cell(r_cell.move_as_ok()); - }, - [&](tonlib_api::tvm_stackEntryCell& cell) { - auto r_cell = vm::std_boc_deserialize(cell.cell_->bytes_); - if (r_cell.is_error()) { - status = r_cell.move_as_error(); - return; - } - stack.write().push_cell(r_cell.move_as_ok()); - }, - [&](tonlib_api::tvm_stackEntryNumber& number) { - [&](tonlib_api::tvm_numberDecimal& dec) { - auto num = td::dec_string_to_int256(dec.number_); - if (num.is_null()) { - status = td::Status::Error("Failed to parse dec string to int256"); - return; - } - stack.write().push_int(std::move(num)); - }(*number.number_); - })); + TRY_RESULT(e, from_tonlib_api(*entry)); + stack.write().push(std::move(e)); } - TRY_STATUS(std::move(status)); args.set_stack(std::move(stack)); auto res = smc->run_get_method(std::move(args)); // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; std::vector> res_stack; for (auto& entry : res.stack->as_span()) { - switch (entry.type()) { - case vm::StackEntry::Type::t_int: - res_stack.push_back(tonlib_api::make_object( - tonlib_api::make_object(dec_string(entry.as_int())))); - break; - case vm::StackEntry::Type::t_slice: - res_stack.push_back( - tonlib_api::make_object(tonlib_api::make_object( - to_bytes(vm::CellBuilder().append_cellslice(entry.as_slice()).finalize())))); - break; - case vm::StackEntry::Type::t_cell: - res_stack.push_back(tonlib_api::make_object( - tonlib_api::make_object(to_bytes(entry.as_cell())))); - break; - default: - res_stack.push_back(tonlib_api::make_object()); - break; - } + res_stack.push_back(to_tonlib_api(entry)); } promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); return td::Status::OK(); @@ -2576,11 +2638,6 @@ td::Status TonlibClient::do_request(const tonlib_api::unpackAccountAddress& requ return TonlibError::Internal(); } template -td::Status TonlibClient::do_request(const tonlib_api::options_validateConfig& request, P&&) { - UNREACHABLE(); - return TonlibError::Internal(); -} -template td::Status TonlibClient::do_request(tonlib_api::getBip39Hints& request, P&&) { UNREACHABLE(); return TonlibError::Internal(); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 68936a43..f624440a 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -58,6 +58,14 @@ class TonlibClient : public td::actor::Actor { ~TonlibClient(); + struct FullConfig { + Config config; + bool use_callbacks_for_network; + LastBlockState last_state; + std::string last_state_key; + td::uint32 wallet_id; + }; + private: enum class State { Uninited, Running, Closed } state_ = State::Uninited; td::unique_ptr callback_; @@ -66,8 +74,7 @@ class TonlibClient : public td::actor::Actor { Config config_; td::uint32 config_generation_{0}; td::uint32 wallet_id_; - std::string blockchain_name_; - bool ignore_cache_{false}; + std::string last_state_key_; bool use_callbacks_for_network_{false}; // KeyStorage @@ -89,7 +96,7 @@ class TonlibClient : public td::actor::Actor { ExtClientRef get_client_ref(); void init_ext_client(); - void init_last_block(td::optional o_master_config); + void init_last_block(LastBlockState state); void init_last_config(); bool is_closing_{false}; @@ -128,7 +135,6 @@ class TonlibClient : public td::actor::Actor { static object_ptr do_static_request(const tonlib_api::testGiver_getAccountAddress& request); static object_ptr do_static_request(const tonlib_api::packAccountAddress& request); static object_ptr do_static_request(const tonlib_api::unpackAccountAddress& request); - static object_ptr do_static_request(tonlib_api::options_validateConfig& request); static object_ptr do_static_request(tonlib_api::getBip39Hints& request); static object_ptr do_static_request(tonlib_api::setLogStream& request); @@ -161,8 +167,6 @@ class TonlibClient : public td::actor::Actor { template td::Status do_request(const tonlib_api::unpackAccountAddress& request, P&&); template - td::Status do_request(const tonlib_api::options_validateConfig& request, P&&); - template td::Status do_request(tonlib_api::getBip39Hints& request, P&&); template @@ -197,18 +201,14 @@ class TonlibClient : public td::actor::Actor { } } - struct FullConfig { - Config config; - td::optional o_master_config; - bool use_callbacks_for_network; - bool ignore_cache; - td::uint32 wallet_id; - }; - static td::Result validate_config(tonlib_api::object_ptr config); + td::Result validate_config(tonlib_api::object_ptr config); void set_config(FullConfig config); - td::Status do_request(const tonlib_api::init& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::init& request, td::Promise>&& promise); td::Status do_request(const tonlib_api::close& request, td::Promise>&& promise); - td::Status do_request(tonlib_api::options_setConfig& request, td::Promise>&& promise); + td::Status do_request(tonlib_api::options_validateConfig& request, + td::Promise>&& promise); + td::Status do_request(tonlib_api::options_setConfig& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::raw_sendMessage& request, td::Promise>&& promise); td::Status do_request(const tonlib_api::raw_createAndSendMessage& request, diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 716fae30..fec9f86c 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -2,6 +2,7 @@ #include "td/utils/filesystem.h" #include "td/utils/OptionsParser.h" +#include "td/utils/overloaded.h" #include "td/utils/Parser.h" #include "td/utils/port/signals.h" #include "td/utils/port/path.h" @@ -177,10 +178,6 @@ class TonlibCli : public td::actor::Actor { ? make_object(options_.config, options_.name, options_.use_callbacks_for_network, options_.ignore_cache) : nullptr; - auto config2 = !options_.config.empty() - ? make_object(options_.config, options_.name, - options_.use_callbacks_for_network, options_.ignore_cache) - : nullptr; tonlib_api::object_ptr ks_type; if (options_.in_memory) { @@ -188,18 +185,13 @@ class TonlibCli : public td::actor::Actor { } else { ks_type = make_object(options_.key_dir); } - auto obj = - tonlib::TonlibClient::static_request(make_object(std::move(config2))); - if (obj->get_id() != tonlib_api::error::ID) { - auto info = ton::move_tl_object_as(obj); - wallet_id_ = static_cast(info->default_wallet_id_); - } else { - LOG(ERROR) << "Invalid config"; - } send_query(make_object(make_object(std::move(config), std::move(ks_type))), - [](auto r_ok) { + [&](auto r_ok) { LOG_IF(ERROR, r_ok.is_error()) << r_ok.error(); - td::TerminalIO::out() << "Tonlib is inited\n"; + if (r_ok.is_ok()) { + wallet_id_ = static_cast(r_ok.ok()->config_info_->default_wallet_id_); + td::TerminalIO::out() << "Tonlib is inited\n"; + } }); if (options_.one_shot) { td::actor::send_closure(actor_id(this), &TonlibCli::parse_line, td::BufferSlice(options_.cmd)); @@ -482,6 +474,73 @@ class TonlibCli : public td::actor::Actor { tonlib_api::make_object(dec_string(num))); } + td::Result>> parse_stack(td::ConstParser& parser, + td::Slice end_token) { + std::vector> stack; + while (true) { + auto word = parser.read_word(); + LOG(ERROR) << word << " vs " << end_token; + if (word == end_token) { + break; + } + if (word == "[") { + TRY_RESULT(elements, parse_stack(parser, "]")); + stack.push_back(tonlib_api::make_object( + tonlib_api::make_object(std::move(elements)))); + } else if (word == "(") { + TRY_RESULT(elements, parse_stack(parser, ")")); + stack.push_back(tonlib_api::make_object( + tonlib_api::make_object(std::move(elements)))); + } else { + TRY_RESULT(stack_entry, parse_stack_entry(word)); + stack.push_back(std::move(stack_entry)); + } + } + return std::move(stack); + } + + static void store_entry(td::StringBuilder& sb, tonlib_api::tvm_StackEntry& entry) { + downcast_call(entry, td::overloaded( + [&](tonlib_api::tvm_stackEntryCell& cell) { + auto r_cell = vm::std_boc_deserialize(cell.cell_->bytes_); + if (r_cell.is_error()) { + sb << ""; + } + auto cs = vm::load_cell_slice(r_cell.move_as_ok()); + std::stringstream ss; + cs.print_rec(ss); + sb << ss.str(); + }, + [&](tonlib_api::tvm_stackEntrySlice& cell) { + auto r_cell = vm::std_boc_deserialize(cell.slice_->bytes_); + if (r_cell.is_error()) { + sb << ""; + } + auto cs = vm::load_cell_slice(r_cell.move_as_ok()); + std::stringstream ss; + cs.print_rec(ss); + sb << ss.str(); + }, + [&](tonlib_api::tvm_stackEntryNumber& cell) { sb << cell.number_->number_; }, + [&](tonlib_api::tvm_stackEntryTuple& cell) { + sb << "("; + for (auto& element : cell.tuple_->elements_) { + sb << " "; + store_entry(sb, *element); + } + sb << " )"; + }, + [&](tonlib_api::tvm_stackEntryList& cell) { + sb << "["; + for (auto& element : cell.list_->elements_) { + sb << " "; + store_entry(sb, *element); + } + sb << " ]"; + }, + [&](tonlib_api::tvm_stackEntryUnsupported& cell) { sb << ""; })); + } + void run_method(td::ConstParser& parser, td::Promise promise) { TRY_RESULT_PROMISE(promise, addr, to_account_address(parser.read_word(), false)); @@ -492,15 +551,15 @@ class TonlibCli : public td::actor::Actor { } else { method = tonlib_api::make_object(method_str.str()); } - std::vector> stack; - while (true) { - auto word = parser.read_word(); - if (word.empty()) { - break; - } - TRY_RESULT_PROMISE(promise, stack_entry, parse_stack_entry(word)); - stack.push_back(std::move(stack_entry)); + TRY_RESULT_PROMISE(promise, stack, parse_stack(parser, "")); + td::StringBuilder sb; + for (auto& entry : stack) { + store_entry(sb, *entry); + sb << "\n"; } + + td::TerminalIO::out() << "Run " << to_string(method) << "With stack:\n" << sb.as_cslice(); + auto to_run = tonlib_api::make_object(0 /*fixme*/, std::move(method), std::move(stack)); @@ -513,9 +572,16 @@ class TonlibCli : public td::actor::Actor { to_run->id_ = info->id_; send_query(std::move(to_run), promise.send_closure(actor_id(this), &TonlibCli::run_method_3)); } - void run_method_3(tonlib_api::object_ptr info, td::Promise promise) { - td::TerminalIO::out() << "Got smc result " << to_string(info); + td::StringBuilder sb; + for (auto& entry : info->stack_) { + store_entry(sb, *entry); + sb << "\n"; + } + + td::TerminalIO::out() << "Got smc result. exit code: " << info->exit_code_ << ", gas_used: " << info->gas_used_ + << "\n" + << sb.as_cslice(); promise.set_value({}); } @@ -972,7 +1038,7 @@ class TonlibCli : public td::actor::Actor { void import_key(std::vector words, td::Slice password) { using tonlib_api::make_object; - send_query(make_object(td::SecureString(password), td::SecureString(), + send_query(make_object(td::SecureString(password), td::SecureString(" test mnemonic"), make_object(std::move(words))), [this, password = td::SecureString(password)](auto r_res) { if (r_res.is_error()) { @@ -1147,7 +1213,7 @@ class TonlibCli : public td::actor::Actor { return; } td::TerminalIO::out() << to_string(r_res.ok()); - self->on_ok(); + //self->on_ok(); }); self->send_query(make_object(r_res.ok()->id_), [self](auto r_res) { @@ -1160,7 +1226,7 @@ class TonlibCli : public td::actor::Actor { self->on_ok(); }); - self->on_ok(); + //self->on_ok(); }); } diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index c4207bbc..82351ae7 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -14,3 +14,8 @@ add_executable(json2tlo json2tlo.cpp ) target_link_libraries(json2tlo tl_api ton_crypto keys ) target_include_directories(json2tlo PUBLIC $/..) + +add_executable(pack-viewer pack-viewer.cpp ) +target_link_libraries(pack-viewer tl_api ton_crypto keys validator tddb ) +target_include_directories(pack-viewer PUBLIC + $/..) diff --git a/utils/pack-viewer.cpp b/utils/pack-viewer.cpp new file mode 100644 index 00000000..3897bf18 --- /dev/null +++ b/utils/pack-viewer.cpp @@ -0,0 +1,61 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2019 Telegram Systems LLP +*/ +#include +#include +#include +#include +#include +#include "td/utils/OptionsParser.h" + +#include "validator/db/package.hpp" +#include "validator/db/fileref.hpp" + +void run(std::string filename) { + auto R = ton::Package::open(filename, true, false); + if (R.is_error()) { + std::cerr << "failed to open archive '" << filename << "': " << R.move_as_error().to_string(); + std::_Exit(2); + } + auto p = R.move_as_ok(); + + p.iterate([&](std::string filename, td::BufferSlice data, td::uint64 offset) -> bool { + auto E = ton::validator::FileReference::create(filename); + if (E.is_error()) { + std::cout << "bad filename\n"; + } else { + std::cout << filename << " " << data.size() << "\n"; + } + return true; + }); +} + +int main(int argc, char **argv) { + run(argv[1]); + return 0; +} + diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 67171a1e..542ecf00 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -43,6 +43,7 @@ #include "td/utils/port/path.h" #include "td/utils/port/signals.h" #include "td/utils/port/user.h" +#include "td/utils/port/rlimit.h" #include "td/utils/ThreadSafeCounter.h" #include "td/utils/TsFileLog.h" #include "td/utils/Random.h" @@ -2842,6 +2843,8 @@ int main(int argc, char *argv[]) { td::log_interface = td::default_log_interface; }; + LOG_STATUS(td::change_maximize_rlimit(td::RlimitType::nofile, 65536)); + std::vector> acts; td::OptionsParser p; diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 73cf0c99..564e21ef 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -10,17 +10,17 @@ add_subdirectory(impl) set(VALIDATOR_DB_SOURCE db/archiver.cpp db/archiver.hpp - db/archive-db.cpp - db/archive-db.hpp + db/archive-manager.cpp + db/archive-manager.hpp + db/archive-slice.cpp + db/archive-slice.hpp db/blockdb.cpp db/blockdb.hpp db/celldb.cpp db/celldb.hpp db/files-async.hpp - db/filedb.hpp - db/filedb.cpp - db/ltdb.hpp - db/ltdb.cpp + db/fileref.hpp + db/fileref.cpp db/rootdb.cpp db/rootdb.hpp db/statedb.hpp @@ -52,6 +52,8 @@ set(VALIDATOR_HEADERS interfaces/validator-manager.h interfaces/validator-set.h invariants.hpp + + import-db-slice.hpp manager-disk.h manager-disk.hpp @@ -69,6 +71,7 @@ set(VALIDATOR_SOURCE apply-block.cpp block-handle.cpp get-next-key-blocks.cpp + import-db-slice.cpp shard-client.cpp state-serializer.cpp token-manager.cpp @@ -124,6 +127,8 @@ set(FULL_NODE_SOURCE net/download-block.cpp net/download-block-new.hpp net/download-block-new.cpp + net/download-archive-slice.hpp + net/download-archive-slice.cpp net/download-next-block.hpp net/download-next-block.cpp net/download-state.hpp diff --git a/validator/apply-block.cpp b/validator/apply-block.cpp index b5dfbd9d..82949285 100644 --- a/validator/apply-block.cpp +++ b/validator/apply-block.cpp @@ -53,6 +53,10 @@ void ApplyBlock::alarm() { void ApplyBlock::start_up() { VLOG(VALIDATOR_DEBUG) << "running apply_block for " << id_; + if (id_.is_masterchain()) { + masterchain_block_id_ = id_; + } + alarm_timestamp() = timeout_; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -144,6 +148,7 @@ void ApplyBlock::got_block_handle(BlockHandle handle) { } void ApplyBlock::written_block_data() { + VLOG(VALIDATOR_DEBUG) << "apply block: written block data for " << id_; if (handle_->is_applied() && handle_->processed()) { finish_query(); } else { @@ -161,6 +166,7 @@ void ApplyBlock::written_block_data() { } void ApplyBlock::got_cur_state(td::Ref state) { + VLOG(VALIDATOR_DEBUG) << "apply block: received state for " << id_; state_ = std::move(state); CHECK(handle_->received_state()); written_state(); @@ -171,6 +177,7 @@ void ApplyBlock::written_state() { finish_query(); return; } + VLOG(VALIDATOR_DEBUG) << "apply block: setting next for parents of " << id_; if (handle_->id().id.seqno != 0 && !handle_->is_applied()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -201,10 +208,12 @@ void ApplyBlock::written_next() { return; } - if (handle_->id().id.seqno != 0) { + VLOG(VALIDATOR_DEBUG) << "apply block: applying parents of " << id_; + + if (handle_->id().id.seqno != 0 && !handle_->is_applied()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { - td::actor::send_closure(SelfId, &ApplyBlock::abort_query, R.move_as_error()); + td::actor::send_closure(SelfId, &ApplyBlock::abort_query, R.move_as_error_prefix("prev: ")); } else { td::actor::send_closure(SelfId, &ApplyBlock::applied_prev); } @@ -213,9 +222,13 @@ void ApplyBlock::written_next() { td::MultiPromise mp; auto g = mp.init_guard(); g.add_promise(std::move(P)); - run_apply_block_query(handle_->one_prev(true), td::Ref{}, manager_, timeout_, g.get_promise()); + BlockIdExt m = masterchain_block_id_; + if (id_.is_masterchain()) { + m = id_; + } + run_apply_block_query(handle_->one_prev(true), td::Ref{}, m, manager_, timeout_, g.get_promise()); if (handle_->merge_before()) { - run_apply_block_query(handle_->one_prev(false), td::Ref{}, manager_, timeout_, g.get_promise()); + run_apply_block_query(handle_->one_prev(false), td::Ref{}, m, manager_, timeout_, g.get_promise()); } } else { applied_prev(); @@ -223,6 +236,10 @@ void ApplyBlock::written_next() { } void ApplyBlock::applied_prev() { + VLOG(VALIDATOR_DEBUG) << "apply block: waiting manager's confirm for " << id_; + if (!id_.is_masterchain()) { + handle_->set_masterchain_ref_block(masterchain_block_id_.seqno()); + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &ApplyBlock::abort_query, R.move_as_error()); @@ -234,7 +251,12 @@ void ApplyBlock::applied_prev() { } void ApplyBlock::applied_set() { + VLOG(VALIDATOR_DEBUG) << "apply block: setting apply bit for " << id_; handle_->set_applied(); + if (handle_->id().seqno() > 0) { + CHECK(handle_->handle_moved_to_archive()); + CHECK(handle_->moved_to_archive()); + } if (handle_->need_flush()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { diff --git a/validator/apply-block.hpp b/validator/apply-block.hpp index e59c558e..b64ccce1 100644 --- a/validator/apply-block.hpp +++ b/validator/apply-block.hpp @@ -39,9 +39,14 @@ namespace validator { class ApplyBlock : public td::actor::Actor { public: - ApplyBlock(BlockIdExt id, td::Ref block, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise) - : id_(id), block_(std::move(block)), manager_(manager), timeout_(timeout), promise_(std::move(promise)) { + ApplyBlock(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, + td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) + : id_(id) + , block_(std::move(block)) + , masterchain_block_id_(masterchain_block_id) + , manager_(manager) + , timeout_(timeout) + , promise_(std::move(promise)) { } static constexpr td::uint32 apply_block_priority() { @@ -65,6 +70,7 @@ class ApplyBlock : public td::actor::Actor { private: BlockIdExt id_; td::Ref block_; + BlockIdExt masterchain_block_id_; td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise promise_; diff --git a/validator/block-handle.cpp b/validator/block-handle.cpp index 7c71630b..e1b0e514 100644 --- a/validator/block-handle.cpp +++ b/validator/block-handle.cpp @@ -32,18 +32,19 @@ void BlockHandleImpl::flush(td::actor::ActorId manage td::BufferSlice BlockHandleImpl::serialize() const { while (locked()) { } - auto flags = flags_.load(std::memory_order_consume) & ~Flags::dbf_processed; + auto flags = flags_.load(std::memory_order_consume) & ~(Flags::dbf_processed | Flags::dbf_moved_handle); return create_serialize_tl_object( create_tl_block_id(id_), flags, (flags & dbf_inited_prev_left) ? create_tl_block_id(prev_[0]) : nullptr, (flags & dbf_inited_prev_right) ? create_tl_block_id(prev_[1]) : nullptr, (flags & dbf_inited_next_left) ? create_tl_block_id(next_[0]) : nullptr, (flags & dbf_inited_next_right) ? create_tl_block_id(next_[1]) : nullptr, (flags & dbf_inited_lt) ? lt_ : 0, - (flags & dbf_inited_ts) ? ts_ : 0, (flags & dbf_inited_state) ? state_ : RootHash::zero()); + (flags & dbf_inited_ts) ? ts_ : 0, (flags & dbf_inited_state) ? state_ : RootHash::zero(), + (flags & dbf_inited_masterchain_ref_block) ? masterchain_ref_seqno_ : 0); } BlockHandleImpl::BlockHandleImpl(td::BufferSlice data) { auto obj = fetch_tl_object(std::move(data), true).move_as_ok(); - flags_ = obj->flags_ & ~Flags::dbf_processed; + flags_ = obj->flags_ & ~(Flags::dbf_processed | Flags::dbf_moved_handle); id_ = create_block_id(obj->id_); prev_[0] = (flags_ & dbf_inited_prev_left) ? create_block_id(obj->prev_left_) : BlockIdExt{}; prev_[1] = (flags_ & dbf_inited_prev_right) ? create_block_id(obj->prev_right_) : BlockIdExt{}; @@ -52,6 +53,8 @@ BlockHandleImpl::BlockHandleImpl(td::BufferSlice data) { lt_ = (flags_ & dbf_inited_lt) ? obj->lt_ : 0; ts_ = (flags_ & dbf_inited_ts) ? obj->ts_ : 0; state_ = (flags_ & dbf_inited_state) ? obj->state_ : RootHash::zero(); + masterchain_ref_seqno_ = + (flags_ & dbf_inited_masterchain_ref_block) ? static_cast(obj->masterchain_ref_seqno_) : 0; get_thread_safe_counter().add(1); } diff --git a/validator/block-handle.hpp b/validator/block-handle.hpp index 3cbaf2e1..06bd1613 100644 --- a/validator/block-handle.hpp +++ b/validator/block-handle.hpp @@ -56,11 +56,12 @@ struct BlockHandleImpl : public BlockHandleInterface { dbf_inited_state_boc = 0x100000, dbf_archived = 0x200000, dbf_applied = 0x400000, - dbf_moved = 0x1000000, + dbf_inited_masterchain_ref_block = 0x800000, dbf_deleted = 0x2000000, dbf_deleted_boc = 0x4000000, dbf_moved_new = 0x8000000, dbf_processed = 0x10000000, + dbf_moved_handle = 0x20000000, }; std::atomic version_{0}; @@ -72,6 +73,7 @@ struct BlockHandleImpl : public BlockHandleInterface { LogicalTime lt_; UnixTime ts_; RootHash state_; + BlockSeqno masterchain_ref_seqno_; static constexpr td::uint64 lock_const() { return static_cast(1) << 32; @@ -93,12 +95,12 @@ struct BlockHandleImpl : public BlockHandleInterface { bool received() const override { return flags_.load(std::memory_order_consume) & Flags::dbf_received; } - bool moved_to_storage() const override { - return flags_.load(std::memory_order_consume) & Flags::dbf_moved; - } bool moved_to_archive() const override { return flags_.load(std::memory_order_consume) & Flags::dbf_moved_new; } + bool handle_moved_to_archive() const override { + return flags_.load(std::memory_order_consume) & Flags::dbf_moved_handle; + } bool deleted() const override { return flags_.load(std::memory_order_consume) & Flags::dbf_deleted; } @@ -199,6 +201,13 @@ struct BlockHandleImpl : public BlockHandleInterface { bool is_applied() const override { return flags_.load(std::memory_order_consume) & Flags::dbf_applied; } + bool inited_masterchain_ref_block() const override { + return id_.is_masterchain() || (flags_.load(std::memory_order_consume) & Flags::dbf_inited_masterchain_ref_block); + } + BlockSeqno masterchain_ref_block() const override { + CHECK(inited_masterchain_ref_block()); + return id_.is_masterchain() ? id_.seqno() : masterchain_ref_seqno_; + } std::vector prev() const override { if (is_zero()) { return {}; @@ -389,23 +398,17 @@ struct BlockHandleImpl : public BlockHandleInterface { flags_ |= Flags::dbf_received; unlock(); } - void set_moved_to_storage() override { - if (flags_.load(std::memory_order_consume) & Flags::dbf_moved) { - return; - } - lock(); - flags_ |= Flags::dbf_moved; - unlock(); - } void set_moved_to_archive() override { if (flags_.load(std::memory_order_consume) & Flags::dbf_moved_new) { return; } lock(); flags_ |= Flags::dbf_moved_new; - flags_ &= ~Flags::dbf_moved; unlock(); } + void set_handle_moved_to_archive() override { + flags_ |= Flags::dbf_moved_handle; + } void set_deleted() override { if (flags_.load(std::memory_order_consume) & Flags::dbf_deleted) { return; @@ -485,6 +488,14 @@ struct BlockHandleImpl : public BlockHandleInterface { unlock(); } } + void set_masterchain_ref_block(BlockSeqno seqno) override { + if (!inited_masterchain_ref_block()) { + lock(); + masterchain_ref_seqno_ = seqno; + flags_ |= Flags::dbf_inited_masterchain_ref_block; + unlock(); + } + } void unsafe_clear_applied() override { if (is_applied()) { diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp new file mode 100644 index 00000000..72d56e56 --- /dev/null +++ b/validator/db/archive-manager.cpp @@ -0,0 +1,986 @@ +#include "archive-manager.hpp" +#include "td/actor/MultiPromise.h" +#include "td/utils/overloaded.h" +#include "files-async.hpp" +#include "td/db/RocksDb.h" +#include "common/delay.h" + +namespace ton { + +namespace validator { + +std::string PackageId::path() const { + if (temp) { + return "files/packages/"; + } else if (key) { + char s[24]; + sprintf(s, "key%03d", id / 1000000); + return PSTRING() << "archive/packages/" << s << "/"; + } else { + char s[20]; + sprintf(s, "arch%04d", id / 100000); + return PSTRING() << "archive/packages/" << s << "/"; + } +} + +std::string PackageId::name() const { + if (temp) { + return PSTRING() << "temp.archive." << id; + } else if (key) { + char s[20]; + sprintf(s, "%06d", id); + return PSTRING() << "key.archive." << s; + } else { + char s[10]; + sprintf(s, "%05d", id); + return PSTRING() << "archive." << s; + } +} + +ArchiveManager::ArchiveManager(td::actor::ActorId root, std::string db_root) : db_root_(db_root) { +} + +void ArchiveManager::add_handle(BlockHandle handle, td::Promise promise) { + if (handle->handle_moved_to_archive()) { + update_handle(std::move(handle), std::move(promise)); + return; + } + auto p = get_package_id_force(handle->masterchain_ref_block(), handle->id().shard_full(), handle->id().seqno(), + handle->unix_time(), handle->logical_time(), + handle->inited_is_key_block() && handle->is_key_block()); + auto f = get_file_desc(handle->id().shard_full(), p, handle->id().seqno(), handle->unix_time(), + handle->logical_time(), true); + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_handle, std::move(handle), std::move(promise)); +} + +void ArchiveManager::update_handle(BlockHandle handle, td::Promise promise) { + FileDescription *f; + if (handle->handle_moved_to_archive()) { + CHECK(handle->inited_unix_time()); + f = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), handle->id().seqno(), + handle->unix_time(), handle->logical_time(), true); + } else { + f = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true); + } + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::update_handle, std::move(handle), std::move(promise)); +} + +void ArchiveManager::add_file(BlockHandle handle, FileReference ref_id, td::BufferSlice data, + td::Promise promise) { + bool copy_to_key = false; + if (handle->inited_is_key_block() && handle->is_key_block() && handle->inited_unix_time() && + handle->inited_logical_time() && handle->inited_masterchain_ref_block()) { + copy_to_key = (ref_id.ref().get_offset() == ref_id.ref().offset()) || + (ref_id.ref().get_offset() == ref_id.ref().offset()); + } + if (!handle->handle_moved_to_archive()) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + auto f1 = get_file_desc(handle->id().shard_full(), get_temp_package_id(), 0, 0, 0, true); + td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), data.clone(), + ig.get_promise()); + if (copy_to_key) { + auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()), + handle->id().seqno(), handle->unix_time(), handle->logical_time(), true); + td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, ref_id, std::move(data), ig.get_promise()); + } + return; + } + + CHECK(handle->inited_is_key_block()); + + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + auto f1 = get_file_desc(handle->id().shard_full(), get_package_id(handle->masterchain_ref_block()), + handle->id().seqno(), handle->unix_time(), handle->logical_time(), true); + td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::add_file, ref_id, data.clone(), ig.get_promise()); + if (copy_to_key) { + auto f2 = get_file_desc(handle->id().shard_full(), get_key_package_id(handle->masterchain_ref_block()), + handle->id().seqno(), handle->unix_time(), handle->logical_time(), true); + td::actor::send_closure(f2->file_actor_id(), &ArchiveSlice::add_file, ref_id, std::move(data), ig.get_promise()); + } +} + +void ArchiveManager::add_key_block_proof(UnixTime ts, BlockSeqno seqno, LogicalTime lt, FileReference ref_id, + td::BufferSlice data, td::Promise promise) { + auto f = get_file_desc(ShardIdFull{masterchainId}, get_key_package_id(seqno), seqno, ts, lt, true); + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), std::move(data), + std::move(promise)); +} + +void ArchiveManager::add_temp_file_short(FileReference ref_id, td::BufferSlice data, td::Promise promise) { + auto f = get_file_desc(ref_id.shard(), get_temp_package_id(), 0, 0, 0, true); + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::add_file, std::move(ref_id), std::move(data), + std::move(promise)); +} + +void ArchiveManager::get_handle(BlockIdExt block_id, td::Promise promise) { + auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), false); + if (f) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, idx = get_max_temp_file_desc_idx(), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok()) { + promise.set_value(R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &ArchiveManager::get_handle_cont, block_id, idx, std::move(promise)); + } + }); + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_handle, block_id, std::move(P)); + } else { + get_handle_cont(block_id, get_max_temp_file_desc_idx(), std::move(promise)); + } +} + +void ArchiveManager::get_handle_cont(BlockIdExt block_id, PackageId idx, td::Promise promise) { + if (idx.is_empty()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "block handle not in db")); + return; + } + auto f = get_temp_file_desc_by_idx(idx); + if (!f) { + promise.set_error(td::Status::Error(ErrorCode::notready, "block handle not in db")); + return; + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, idx = get_prev_temp_file_desc_idx(idx), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ArchiveManager::get_handle_finish, R.move_as_ok(), std::move(promise)); + } else { + td::actor::send_closure(SelfId, &ArchiveManager::get_handle_cont, block_id, idx, std::move(promise)); + } + }); + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_handle, block_id, std::move(P)); +} + +void ArchiveManager::get_handle_finish(BlockHandle handle, td::Promise promise) { + auto f = get_file_desc_by_seqno(handle->id().shard_full(), handle->id().seqno(), false); + if (!f) { + promise.set_value(std::move(handle)); + return; + } + auto P = td::PromiseCreator::lambda([handle, promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok()) { + promise.set_value(R.move_as_ok()); + } else { + promise.set_value(std::move(handle)); + } + }); + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_handle, handle->id(), std::move(P)); +} + +void ArchiveManager::get_file_short(FileReference ref_id, td::Promise promise) { + bool search_in_key = false; + BlockIdExt block_id; + ref_id.ref().visit(td::overloaded( + [&](const fileref::Proof &p) { + search_in_key = p.block_id.is_masterchain(); + block_id = p.block_id; + }, + [&](const fileref::ProofLink &p) { + search_in_key = p.block_id.is_masterchain(); + block_id = p.block_id; + }, + [&](const auto &p) {})); + if (search_in_key) { + auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), true); + if (f) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok()) { + promise.set_value(R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &ArchiveManager::get_temp_file_short, std::move(ref_id), std::move(promise)); + } + }); + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, ref_id, std::move(P)); + return; + } + } + get_temp_file_short(std::move(ref_id), std::move(promise)); +} + +void ArchiveManager::get_key_block_proof(FileReference ref_id, td::Promise promise) { + bool search_in_key = false; + BlockIdExt block_id; + ref_id.ref().visit(td::overloaded( + [&](const fileref::Proof &p) { + search_in_key = p.block_id.is_masterchain(); + block_id = p.block_id; + }, + [&](const fileref::ProofLink &p) { + search_in_key = p.block_id.is_masterchain(); + block_id = p.block_id; + }, + [&](const auto &p) {})); + if (search_in_key) { + auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), true); + if (f) { + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, ref_id, std::move(promise)); + } else { + promise.set_error(td::Status::Error(ErrorCode::notready, "key proof not in db")); + } + } else { + promise.set_error( + td::Status::Error(ErrorCode::protoviolation, "only proof/prooflink supported in get_key_block_proof")); + } +} + +void ArchiveManager::get_temp_file_short(FileReference ref_id, td::Promise promise) { + get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise)); +} + +void ArchiveManager::get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise promise) { + auto f = get_temp_file_desc_by_idx(idx); + if (!f) { + promise.set_error(td::Status::Error(ErrorCode::notready, "file not in db")); + return; + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id, idx = get_prev_temp_file_desc_idx(idx), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok()) { + promise.set_value(R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &ArchiveManager::get_file_short_cont, std::move(ref_id), idx, std::move(promise)); + } + }); + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, std::move(ref_id), std::move(P)); +} + +void ArchiveManager::get_file(BlockHandle handle, FileReference ref_id, td::Promise promise) { + if (handle->moved_to_archive()) { + auto f = get_file_desc(handle->id().shard_full(), PackageId{handle->masterchain_ref_block(), false, false}, 0, 0, 0, + false); + if (f) { + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_file, std::move(ref_id), std::move(promise)); + return; + } + } + get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise)); +} + +void ArchiveManager::written_perm_state(FileReferenceShort id) { + perm_states_.emplace(id.hash(), id); +} + +void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise) { + auto id = FileReference{fileref::ZeroState{block_id}}; + auto hash = id.hash(); + if (perm_states_.find(hash) != perm_states_.end()) { + promise.set_value(td::Unit()); + return; + } + + auto path = db_root_ + "/archive/states/" + id.filename_short(); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), id = id.shortref(), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id); + promise.set_value(td::Unit()); + } + }); + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", path, std::move(data), std::move(P)) + .release(); +} + +void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, + td::Promise promise) { + auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + auto hash = id.hash(); + if (perm_states_.find(hash) != perm_states_.end()) { + promise.set_value(td::Unit()); + return; + } + + auto path = db_root_ + "/archive/states/" + id.filename_short(); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), id = id.shortref(), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id); + promise.set_value(td::Unit()); + } + }); + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", path, std::move(data), std::move(P)) + .release(); +} + +void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise promise) { + auto id = FileReference{fileref::ZeroState{block_id}}; + auto hash = id.hash(); + if (perm_states_.find(hash) == perm_states_.end()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "zerostate not in db")); + return; + } + + auto path = db_root_ + "/archive/states/" + id.filename_short(); + td::actor::create_actor("readfile", path, 0, -1, 0, std::move(promise)).release(); +} + +void ArchiveManager::check_zero_state(BlockIdExt block_id, td::Promise promise) { + auto id = FileReference{fileref::ZeroState{block_id}}; + auto hash = id.hash(); + if (perm_states_.find(hash) == perm_states_.end()) { + promise.set_result(false); + return; + } + promise.set_result(true); +} + +void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + auto hash = id.hash(); + if (perm_states_.find(hash) == perm_states_.end()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); + return; + } + + auto path = db_root_ + "/archive/states/" + id.filename_short(); + td::actor::create_actor("readfile", path, 0, -1, 0, std::move(promise)).release(); +} + +void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, + td::int64 max_size, td::Promise promise) { + auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + auto hash = id.hash(); + if (perm_states_.find(hash) == perm_states_.end()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); + return; + } + + auto path = db_root_ + "/archive/states/" + id.filename_short(); + td::actor::create_actor("readfile", path, offset, max_size, 0, std::move(promise)).release(); +} + +void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + auto hash = id.hash(); + if (perm_states_.find(hash) == perm_states_.end()) { + promise.set_result(false); + return; + } + promise.set_result(true); +} + +void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, + td::Promise promise) { + auto f = get_file_desc_by_unix_time(account_id, ts, false); + if (f) { + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_unix_time, account_id, ts, + std::move(promise)); + } else { + promise.set_error(td::Status::Error(ErrorCode::notready, "ts not in db")); + } +} + +void ArchiveManager::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise) { + auto f = get_file_desc_by_lt(account_id, lt, false); + if (f) { + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(promise)); + } else { + promise.set_error(td::Status::Error(ErrorCode::notready, "lt not in db")); + } +} + +void ArchiveManager::get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, + td::Promise promise) { + auto f = get_file_desc_by_seqno(account_id, seqno, false); + if (f) { + td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_seqno, account_id, seqno, + std::move(promise)); + } else { + promise.set_error(td::Status::Error(ErrorCode::notready, "seqno not in db")); + } +} + +void ArchiveManager::delete_package(PackageId id) { + auto key = create_serialize_tl_object(id.id, id.key, id.temp); + + std::string value; + auto v = index_->get(key.as_slice(), value); + v.ensure(); + CHECK(v.move_as_ok() == td::KeyValue::GetStatus::Ok); + + auto R = fetch_tl_object(value, true); + R.ensure(); + auto x = R.move_as_ok(); + + if (x->deleted_) { + return; + } + + auto &m = get_file_map(id); + auto it = m.find(id); + if (it == m.end() || it->second.deleted) { + return; + } + + it->second.deleted = true; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveManager::deleted_package, id); + }); + td::actor::send_closure(it->second.file_actor_id(), &ArchiveSlice::destroy, std::move(P)); +} + +void ArchiveManager::deleted_package(PackageId id) { + auto key = create_serialize_tl_object(id.id, id.key, id.temp); + + std::string value; + auto v = index_->get(key.as_slice(), value); + v.ensure(); + CHECK(v.move_as_ok() == td::KeyValue::GetStatus::Ok); + + auto R = fetch_tl_object(value, true); + R.ensure(); + auto x = R.move_as_ok(); + + if (x->deleted_) { + return; + } + x->deleted_ = true; + index_->begin_transaction().ensure(); + index_->set(key, serialize_tl_object(x, true)).ensure(); + index_->commit_transaction().ensure(); + + auto &m = get_file_map(id); + auto it = m.find(id); + CHECK(it != m.end()); + CHECK(it->second.deleted); + it->second.clear_actor_id(); +} + +void ArchiveManager::load_package(PackageId id) { + auto key = create_serialize_tl_object(id.id, id.key, id.temp); + + std::string value; + auto v = index_->get(key.as_slice(), value); + v.ensure(); + CHECK(v.move_as_ok() == td::KeyValue::GetStatus::Ok); + + auto R = fetch_tl_object(value, true); + R.ensure(); + auto x = R.move_as_ok(); + + if (x->deleted_) { + return; + } + + FileDescription desc{id, false}; + if (!id.temp) { + for (auto &e : x->firstblocks_) { + desc.first_blocks[ShardIdFull{e->workchain_, static_cast(e->shard_)}] = FileDescription::Desc{ + static_cast(e->seqno_), static_cast(e->unixtime_), static_cast(e->lt_)}; + } + } + + std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); + desc.file = td::actor::create_actor("slice", id.key, id.temp, prefix); + + get_file_map(id).emplace(id, std::move(desc)); +} + +ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, + UnixTime ts, LogicalTime lt, bool force) { + auto &f = get_file_map(id); + auto it = f.find(id); + if (it != f.end()) { + if (it->second.deleted) { + CHECK(!force); + return nullptr; + } + if (force && !id.temp) { + update_desc(it->second, shard, seqno, ts, lt); + } + return &it->second; + } + if (!force) { + return nullptr; + } + + return add_file_desc(shard, id, seqno, ts, lt); +} + +ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, + UnixTime ts, LogicalTime lt) { + auto &f = get_file_map(id); + CHECK(f.count(id) == 0); + + FileDescription desc{id, false}; + td::mkdir(db_root_ + id.path()).ensure(); + std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); + desc.file = td::actor::create_actor("slice", id.key, id.temp, prefix); + if (!id.temp) { + update_desc(desc, shard, seqno, ts, lt); + } + + std::vector> vec; + for (auto &e : desc.first_blocks) { + vec.push_back(create_tl_object(e.first.workchain, e.first.shard, + e.second.seqno, e.second.ts, e.second.lt)); + } + + index_->begin_transaction().ensure(); + // add package info to list of packages + { + std::vector t; + std::vector tk; + std::vector tt; + for (auto &e : files_) { + t.push_back(e.first.id); + } + for (auto &e : key_files_) { + tk.push_back(e.first.id); + } + for (auto &e : temp_files_) { + tt.push_back(e.first.id); + } + (id.temp ? tt : (id.key ? tk : t)).push_back(id.id); + index_ + ->set(create_serialize_tl_object().as_slice(), + create_serialize_tl_object(std::move(t), std::move(tk), std::move(tt)) + .as_slice()) + .ensure(); + } + // add package info key + { + index_ + ->set(create_serialize_tl_object(id.id, id.key, id.temp).as_slice(), + create_serialize_tl_object(id.id, id.key, id.temp, std::move(vec), false) + .as_slice()) + .ensure(); + } + index_->commit_transaction().ensure(); + + return &f.emplace(id, std::move(desc)).first->second; +} + +void ArchiveManager::update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, + LogicalTime lt) { + auto it = desc.first_blocks.find(shard); + if (it != desc.first_blocks.end() && it->second.seqno <= seqno) { + return; + } + desc.first_blocks[shard] = FileDescription::Desc{seqno, ts, lt}; + std::vector> vec; + for (auto &e : desc.first_blocks) { + vec.push_back(create_tl_object(e.first.workchain, e.first.shard, + e.second.seqno, e.second.ts, e.second.lt)); + } + index_->begin_transaction().ensure(); + index_ + ->set(create_serialize_tl_object(desc.id.id, desc.id.key, desc.id.temp).as_slice(), + create_serialize_tl_object(desc.id.id, desc.id.key, desc.id.temp, + std::move(vec), false) + .as_slice()) + .ensure(); + index_->commit_transaction().ensure(); +} + +ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, + bool key_block) { + auto &f = get_file_map(PackageId{0, key_block, false}); + for (auto it = f.rbegin(); it != f.rend(); it++) { + auto i = it->second.first_blocks.find(shard); + if (i != it->second.first_blocks.end() && i->second.seqno <= seqno) { + return &it->second; + } + } + return nullptr; +} + +ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, + bool key_block) { + auto &f = get_file_map(PackageId{0, key_block, false}); + for (auto it = f.rbegin(); it != f.rend(); it++) { + auto i = it->second.first_blocks.find(shard); + if (i != it->second.first_blocks.end() && i->second.ts <= ts) { + return &it->second; + } + } + return nullptr; +} + +ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, + bool key_block) { + auto &f = get_file_map(PackageId{0, key_block, false}); + for (auto it = f.rbegin(); it != f.rend(); it++) { + auto i = it->second.first_blocks.find(shard); + if (i != it->second.first_blocks.end() && i->second.lt <= lt) { + return &it->second; + } + } + return nullptr; +} + +ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, + bool key_block) { + auto &f = get_file_map(PackageId{0, key_block, false}); + if (account.is_masterchain()) { + return get_file_desc_by_seqno(ShardIdFull{masterchainId}, seqno, key_block); + } + for (auto it = f.rbegin(); it != f.rend(); it++) { + bool found = false; + for (int i = 0; i < 60; i++) { + auto shard = shard_prefix(account, i); + auto it2 = it->second.first_blocks.find(shard); + if (it2 != it->second.first_blocks.end()) { + if (it2->second.seqno <= seqno) { + return &it->second; + } + found = true; + } else if (found) { + break; + } + } + } + return nullptr; +} + +ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(AccountIdPrefixFull account, UnixTime ts, + bool key_block) { + auto &f = get_file_map(PackageId{0, key_block, false}); + if (account.is_masterchain()) { + return get_file_desc_by_unix_time(ShardIdFull{masterchainId}, ts, key_block); + } + for (auto it = f.rbegin(); it != f.rend(); it++) { + bool found = false; + for (int i = 0; i < 60; i++) { + auto shard = shard_prefix(account, i); + auto it2 = it->second.first_blocks.find(shard); + if (it2 != it->second.first_blocks.end()) { + if (it2->second.ts <= ts) { + return &it->second; + } + found = true; + } else if (found) { + break; + } + } + } + return nullptr; +} + +ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(AccountIdPrefixFull account, LogicalTime lt, + bool key_block) { + auto &f = get_file_map(PackageId{0, key_block, false}); + if (account.is_masterchain()) { + return get_file_desc_by_lt(ShardIdFull{masterchainId}, lt, key_block); + } + for (auto it = f.rbegin(); it != f.rend(); it++) { + bool found = false; + for (int i = 0; i < 60; i++) { + auto shard = shard_prefix(account, i); + auto it2 = it->second.first_blocks.find(shard); + if (it2 != it->second.first_blocks.end()) { + if (it2->second.lt <= lt) { + return &it->second; + } + found = true; + } else if (found) { + break; + } + } + } + return nullptr; +} + +ArchiveManager::FileDescription *ArchiveManager::get_temp_file_desc_by_idx(PackageId idx) { + auto it = temp_files_.find(idx); + if (it != temp_files_.end()) { + if (it->second.deleted) { + return nullptr; + } else { + return &it->second; + } + } else { + return nullptr; + } +} + +PackageId ArchiveManager::get_max_temp_file_desc_idx() { + if (temp_files_.size() == 0) { + return PackageId::empty(false, true); + } else { + return temp_files_.rbegin()->first; + } +} + +PackageId ArchiveManager::get_prev_temp_file_desc_idx(PackageId idx) { + auto it = temp_files_.lower_bound(idx); + if (it == temp_files_.end()) { + return PackageId::empty(false, true); + } + if (it == temp_files_.begin()) { + return PackageId::empty(false, true); + } + it--; + return it->first; +} + +void ArchiveManager::start_up() { + td::mkdir(db_root_).ensure(); + td::mkdir(db_root_ + "/archive/").ensure(); + td::mkdir(db_root_ + "/archive/tmp/").ensure(); + td::mkdir(db_root_ + "/archive/packages/").ensure(); + td::mkdir(db_root_ + "/archive/states/").ensure(); + td::mkdir(db_root_ + "/files/").ensure(); + td::mkdir(db_root_ + "/files/packages/").ensure(); + index_ = std::make_shared(td::RocksDb::open(db_root_ + "/files/globalindex").move_as_ok()); + std::string value; + auto v = index_->get(create_serialize_tl_object().as_slice(), value); + v.ensure(); + if (v.move_as_ok() == td::KeyValue::GetStatus::Ok) { + auto R = fetch_tl_object(value, true); + R.ensure(); + auto x = R.move_as_ok(); + + for (auto &d : x->packages_) { + load_package(PackageId{static_cast(d), false, false}); + } + for (auto &d : x->key_packages_) { + load_package(PackageId{static_cast(d), true, false}); + } + for (auto &d : x->temp_packages_) { + load_package(PackageId{static_cast(d), false, true}); + } + } + + td::WalkPath::run(db_root_ + "/archive/states/", [&](td::CSlice fname, td::WalkPath::Type t) -> void { + if (t == td::WalkPath::Type::NotDir) { + LOG(ERROR) << "checking file " << fname; + auto pos = fname.rfind('/'); + if (pos != td::Slice::npos) { + fname.remove_prefix(pos + 1); + } + auto R = FileReferenceShort::create(fname.str()); + if (R.is_error()) { + auto R2 = FileReference::create(fname.str()); + if (R2.is_error()) { + LOG(ERROR) << "deleting bad state file '" << fname << "': " << R.move_as_error() << R2.move_as_error(); + td::unlink(db_root_ + "/archive/states/" + fname.str()).ignore(); + return; + } + auto d = R2.move_as_ok(); + auto newfname = d.filename_short(); + td::rename(db_root_ + "/archive/states/" + fname.str(), db_root_ + "/archive/states/" + newfname).ensure(); + R = FileReferenceShort::create(newfname); + R.ensure(); + } + auto f = R.move_as_ok(); + auto hash = f.hash(); + perm_states_[hash] = std::move(f); + } + }).ensure(); + + persistent_state_gc(FileHash::zero()); +} + +void ArchiveManager::run_gc(UnixTime ts) { + auto p = get_temp_package_id_by_unixtime(ts); + std::vector vec; + for (auto &x : temp_files_) { + if (x.first < p) { + vec.push_back(x.first); + } else { + break; + } + } + if (vec.size() <= 1) { + return; + } + vec.resize(vec.size() - 1, PackageId::empty(false, true)); + + for (auto &x : vec) { + delete_package(x); + } +} + +void ArchiveManager::persistent_state_gc(FileHash last) { + if (perm_states_.size() == 0) { + delay_action( + [hash = FileHash::zero(), SelfId = actor_id(this)]() { + td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); + }, + td::Timestamp::in(1.0)); + return; + } + auto it = perm_states_.lower_bound(last); + if (it != perm_states_.end() && it->first == last) { + it++; + } + if (it == perm_states_.end()) { + it = perm_states_.begin(); + } + + auto &F = it->second; + auto hash = F.hash(); + + int res = 0; + BlockSeqno seqno = 0; + F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &x) { res = 1; }, + [&](const fileref::PersistentStateShort &x) { + res = 0; + seqno = x.masterchain_seqno; + }, + [&](const auto &obj) { res = -1; })); + + if (res == -1) { + td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + perm_states_.erase(it); + } + if (res != 0) { + delay_action([hash, SelfId = actor_id( + this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); }, + td::Timestamp::in(1.0)); + return; + } + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), hash](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, nullptr, hash); + } else { + td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, R.move_as_ok(), hash); + } + }); + + get_block_by_seqno(AccountIdPrefixFull{masterchainId, 0}, seqno, std::move(P)); +} + +void ArchiveManager::got_gc_masterchain_handle(BlockHandle handle, FileHash hash) { + bool to_del = false; + if (!handle || !handle->inited_unix_time() || !handle->unix_time()) { + to_del = true; + } else { + auto ttl = ValidatorManager::persistent_state_ttl(handle->unix_time()); + to_del = ttl < td::Clocks::system(); + } + auto it = perm_states_.find(hash); + CHECK(it != perm_states_.end()); + auto &F = it->second; + if (to_del) { + td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + perm_states_.erase(it); + } + delay_action([hash, SelfId = actor_id( + this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); }, + td::Timestamp::in(1.0)); +} + +PackageId ArchiveManager::get_temp_package_id() const { + return get_temp_package_id_by_unixtime(static_cast(td::Clocks::system())); +} + +PackageId ArchiveManager::get_temp_package_id_by_unixtime(UnixTime ts) const { + return PackageId{ts - (ts % 3600), false, true}; +} + +PackageId ArchiveManager::get_key_package_id(BlockSeqno seqno) const { + return PackageId{seqno - seqno % 200000, true, false}; +} + +PackageId ArchiveManager::get_package_id(BlockSeqno seqno) const { + auto it = files_.upper_bound(PackageId{seqno, false, false}); + CHECK(it != files_.begin()); + it--; + return it->first; +} + +PackageId ArchiveManager::get_package_id_force(BlockSeqno masterchain_seqno, ShardIdFull shard, BlockSeqno seqno, + UnixTime ts, LogicalTime lt, bool is_key) { + PackageId p = PackageId::empty(false, false); + if (!is_key) { + auto it = files_.upper_bound(PackageId{masterchain_seqno, false, false}); + p = PackageId{masterchain_seqno - (masterchain_seqno % 20000), false, false}; + if (it != files_.begin()) { + it--; + if (p < it->first) { + p = it->first; + } + } + } else { + p = PackageId{masterchain_seqno, false, false}; + } + auto it = files_.find(p); + if (it != files_.end()) { + return it->first; + } + add_file_desc(shard, p, seqno, ts, lt); + it = files_.find(p); + CHECK(it != files_.end()); + return it->first; +} + +void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { + auto F = get_file_desc_by_seqno(ShardIdFull{masterchainId}, masterchain_seqno, false); + if (!F) { + promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found")); + return; + } + + promise.set_result(F->id.id); +} + +void ArchiveManager::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, + td::Promise promise) { + if (archive_id != static_cast(archive_id)) { + promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found")); + return; + } + auto F = get_file_desc(ShardIdFull{masterchainId}, PackageId{static_cast(archive_id), false, false}, 0, 0, + 0, false); + if (!F) { + promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found")); + return; + } + + td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_slice, offset, limit, std::move(promise)); +} + +void ArchiveManager::commit_transaction() { + if (!async_mode_ || huge_transaction_size_++ >= 100) { + index_->commit_transaction().ensure(); + if (async_mode_) { + huge_transaction_size_ = 0; + huge_transaction_started_ = false; + } + } +} + +void ArchiveManager::set_async_mode(bool mode, td::Promise promise) { + async_mode_ = mode; + if (!async_mode_ && huge_transaction_started_) { + index_->commit_transaction().ensure(); + huge_transaction_size_ = 0; + huge_transaction_started_ = false; + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + + for (auto &x : key_files_) { + if (!x.second.deleted) { + td::actor::send_closure(x.second.file_actor_id(), &ArchiveSlice::set_async_mode, mode, ig.get_promise()); + } + } + for (auto &x : temp_files_) { + if (!x.second.deleted) { + td::actor::send_closure(x.second.file_actor_id(), &ArchiveSlice::set_async_mode, mode, ig.get_promise()); + } + } + for (auto &x : files_) { + if (!x.second.deleted) { + td::actor::send_closure(x.second.file_actor_id(), &ArchiveSlice::set_async_mode, mode, ig.get_promise()); + } + } +} +} // namespace validator + +} // namespace ton diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp new file mode 100644 index 00000000..fc073b6b --- /dev/null +++ b/validator/db/archive-manager.hpp @@ -0,0 +1,155 @@ +#pragma once + +#include "archive-slice.hpp" + +namespace ton { + +namespace validator { + +struct PackageId { + td::uint32 id; + bool key; + bool temp; + + explicit PackageId(td::uint32 id, bool key, bool temp) : id(id), key(key), temp(temp) { + } + + bool operator<(const PackageId &with) const { + return id < with.id; + } + bool operator==(const PackageId &with) const { + return id == with.id; + } + + std::string path() const; + std::string name() const; + + bool is_empty() { + return id == std::numeric_limits::max(); + } + static PackageId empty(bool key, bool temp) { + return PackageId(std::numeric_limits::max(), key, temp); + } +}; + +class RootDb; + +class ArchiveManager : public td::actor::Actor { + public: + ArchiveManager(td::actor::ActorId root, std::string db_root); + + void add_handle(BlockHandle handle, td::Promise promise); + void update_handle(BlockHandle handle, td::Promise promise); + void add_file(BlockHandle handle, FileReference ref_id, td::BufferSlice data, td::Promise promise); + void add_key_block_proof(BlockSeqno seqno, UnixTime ts, LogicalTime lt, FileReference ref_id, td::BufferSlice data, + td::Promise promise); + void add_temp_file_short(FileReference ref_id, td::BufferSlice data, td::Promise promise); + void get_handle(BlockIdExt block_id, td::Promise promise); + void get_key_block_proof(FileReference ref_id, td::Promise promise); + void get_temp_file_short(FileReference ref_id, td::Promise promise); + void get_file_short(FileReference ref_id, td::Promise promise); + void get_file(BlockHandle handle, FileReference ref_id, td::Promise promise); + + void add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise); + void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, + td::Promise promise); + void get_zero_state(BlockIdExt block_id, td::Promise promise); + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, + td::int64 max_size, td::Promise promise); + void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); + void check_zero_state(BlockIdExt block_id, td::Promise promise); + + void run_gc(UnixTime ts); + + /* from LTDB */ + void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise); + void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise); + void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise promise); + + void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise); + void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, + td::Promise promise); + + void start_up() override; + + void begin_transaction(); + void commit_transaction(); + void set_async_mode(bool mode, td::Promise promise); + + private: + struct FileDescription { + struct Desc { + BlockSeqno seqno; + UnixTime ts; + LogicalTime lt; + }; + FileDescription(PackageId id, bool deleted) : id(id), deleted(deleted) { + } + auto file_actor_id() const { + return file.get(); + } + void clear_actor_id() { + file.reset(); + } + PackageId id; + bool deleted; + + std::map first_blocks; + td::actor::ActorOwn file; + }; + + std::map files_; + std::map key_files_; + std::map temp_files_; + bool async_mode_ = false; + bool huge_transaction_started_ = false; + td::uint32 huge_transaction_size_ = 0; + + auto &get_file_map(const PackageId &p) { + return p.key ? key_files_ : p.temp ? temp_files_ : files_; + } + + std::map perm_states_; + + void load_package(PackageId seqno); + void delete_package(PackageId seqno); + void deleted_package(PackageId seqno); + void get_handle_cont(BlockIdExt block_id, PackageId id, td::Promise promise); + void get_handle_finish(BlockHandle handle, td::Promise promise); + void get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise promise); + + FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt, + bool force); + FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt); + void update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, LogicalTime lt); + FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, bool key_block); + FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, bool key_block); + FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, bool key_block); + FileDescription *get_file_desc_by_seqno(AccountIdPrefixFull shard, BlockSeqno seqno, bool key_block); + FileDescription *get_file_desc_by_lt(AccountIdPrefixFull shard, LogicalTime lt, bool key_block); + FileDescription *get_file_desc_by_unix_time(AccountIdPrefixFull shard, UnixTime ts, bool key_block); + FileDescription *get_temp_file_desc_by_idx(PackageId idx); + PackageId get_max_temp_file_desc_idx(); + PackageId get_prev_temp_file_desc_idx(PackageId id); + + void written_perm_state(FileReferenceShort id); + + void persistent_state_gc(FileHash last); + void got_gc_masterchain_handle(BlockHandle handle, FileHash hash); + + std::string db_root_; + + std::shared_ptr index_; + + PackageId get_package_id(BlockSeqno seqno) const; + PackageId get_package_id_force(BlockSeqno masterchain_seqno, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, + LogicalTime lt, bool is_key); + PackageId get_temp_package_id() const; + PackageId get_key_package_id(BlockSeqno seqno) const; + PackageId get_temp_package_id_by_unixtime(UnixTime ts) const; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/db/archive-mover.cpp b/validator/db/archive-mover.cpp new file mode 100644 index 00000000..ec9448ab --- /dev/null +++ b/validator/db/archive-mover.cpp @@ -0,0 +1,475 @@ +#include "archive-mover.hpp" +#include "td/actor/MultiPromise.h" +#include "validator/fabric.h" + +namespace ton { + +namespace validator { + +void ArchiveFileMover::start_up() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); + }); + td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, block_id_, std::move(P)); +} + +void ArchiveFileMover::got_block_handle0(td::Result R) { + if (R.is_ok()) { + handle_ = R.move_as_ok(); + CHECK(handle_->moved_to_archive()); + CHECK(handle_->handle_moved_to_archive()); + finish_query(); + return; + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); + }); + td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read_handle, block_id_, std::move(P)); +} + +void ArchiveFileMover::got_block_handle1(td::Result R) { + if (R.is_ok()) { + handle_ = R.move_as_ok(); + got_block_handle(); + return; + } + + if (R.error().code() != ErrorCode::notready) { + abort_query(R.move_as_error()); + return; + } + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); + }); + td::actor::send_closure(block_db_, &BlockDb::get_block_handle, std::move(P)); +} + +void ArchiveFileMover::got_block_handle2(td::Result R) { + if (R.is_ok()) { + handle_ = R.move_as_ok(); + got_block_handle(); + return; + } + + if (R.error().code() != ErrorCode::notready) { + abort_query(R.move_as_error()); + return; + } + + finish_query(); +} + +void ArchiveFileMover::got_block_handle() { + if (!handle_->is_applied()) { + finish_query(); + return; + } + if (handle_->id().seqno() == 0) { + processed_all_children(); + return; + } + + CHECK(handle_->inited_prev()); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child); + } + }); + + td::actor::create_actor("mover", handle_->one_prev(left_), mode_, block_db_, file_db_, + old_archive_db_, old_archive_manager_, archive_manager_, std::move(P)) + .release(); +} + +void ArchiveFileMover::processed_child() { + if (!left_ || !handle_->merge_before()) { + processed_all_children(); + return; + } + left_ = false; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child); + } + }); + + td::actor::create_actor("mover", handle_->one_prev(left_), mode_, block_db_, file_db_, + old_archive_db_, old_archive_manager_, archive_manager_, std::move(P)) + .release(); +} + +void ArchiveFileMover::processed_all_children() { + if (!handle_->received()) { + got_block_data(td::BufferSlice{}); + } else { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_data, std::move(R)); + }); + + if (handle_->moved_to_archive()) { + CHECK(handle_->inited_unix_time()); + td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), + handle_->is_key_block(), FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P)); + } else { + td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, + FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P)); + } + } +} + +void ArchiveFileMover::got_block_data(td::Result R) { + if (R.is_error()) { + if (R.error().code() != ErrorCode::notready) { + abort_query(R.move_as_error()); + return; + } + } else { + data_ = R.move_as_ok(); + } + if (!handle_->inited_proof()) { + got_block_proof(td::BufferSlice{}); + } else { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof, std::move(R)); + }); + + if (handle_->moved_to_archive()) { + CHECK(handle_->inited_unix_time()); + td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), + handle_->is_key_block(), FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P)); + } else { + td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, + FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P)); + } + } +} + +void ArchiveFileMover::got_block_proof(td::Result R) { + if (R.is_error()) { + if (R.error().code() != ErrorCode::notready) { + abort_query(R.move_as_error()); + return; + } + } else { + proof_ = R.move_as_ok(); + } + if (!handle_->inited_proof_link()) { + got_block_proof_link(td::BufferSlice{}); + } else { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof_link, std::move(R)); + }); + + if (handle_->moved_to_archive()) { + CHECK(handle_->inited_unix_time()); + td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), + handle_->is_key_block(), FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P)); + } else { + td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, + FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P)); + } + } +} + +void ArchiveFileMover::got_block_proof_link(td::Result R) { + if (R.is_error()) { + if (R.error().code() != ErrorCode::notready) { + abort_query(R.move_as_error()); + return; + } + } else { + proof_link_ = R.move_as_ok(); + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveFileMover::written_data); + }); + + if (data_.size() > 0) { + td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Block{block_id_}, + std::move(data_), ig.get_promise()); + } + if (proof_.size() > 0) { + td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Proof{block_id_}, + std::move(proof_), ig.get_promise()); + } + if (proof_link_.size() > 0) { + td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::ProofLink{block_id_}, + std::move(proof_link_), ig.get_promise()); + } +} + +void ArchiveFileMover::written_data() { + handle_->set_moved_to_archive(); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveFileMover::written_handle); + }); + td::actor::send_closure(archive_manager_, &ArchiveManager::add_handle, handle_, std::move(P)); +} + +void ArchiveFileMover::written_handle() { + CHECK(handle_->handle_moved_to_archive()); + finish_query(); +} + +void ArchiveFileMover::abort_query(td::Status error) { + if (promise_) { + promise_.set_error(std::move(error)); + } + stop(); +} + +void ArchiveFileMover::finish_query() { + if (promise_) { + promise_.set_value(td::Unit()); + } + stop(); +} + +void ArchiveKeyBlockMover::start_up() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::skip_block_proof, R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof0); + } + }); + if (proof_link_) { + td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::ProofLink{block_id_}, + std::move(P)); + } else { + td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::Proof{block_id_}, std::move(P)); + } +} + +void ArchiveKeyBlockMover::failed_to_get_proof0() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof1); + } + }); + if (proof_link_) { + td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::ProofLink{block_id_}, + std::move(P)); + } else { + td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::Proof{block_id_}, std::move(P)); + } +} + +void ArchiveKeyBlockMover::failed_to_get_proof1() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof2); + } + }); + if (proof_link_) { + td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P)); + } else { + td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P)); + } +} + +void ArchiveKeyBlockMover::failed_to_get_proof2() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof3); + } + }); + if (proof_link_) { + td::actor::send_closure(file_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P)); + } else { + td::actor::send_closure(file_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P)); + } +} + +void ArchiveKeyBlockMover::failed_to_get_proof3() { + if (proof_link_) { + written_data(); + } else { + proof_link_ = true; + start_up(); + } +} + +void ArchiveKeyBlockMover::got_block_proof(td::BufferSlice data) { + data_ = std::move(data); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::written_data); + }); + + if (proof_link_) { + auto p = create_proof_link(block_id_, data_.clone()).move_as_ok(); + auto h = p->get_basic_header_info().move_as_ok(); + td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime, + fileref::ProofLink{block_id_}, std::move(data_), std::move(P)); + } else { + auto p = create_proof(block_id_, data_.clone()).move_as_ok(); + auto h = p->get_basic_header_info().move_as_ok(); + td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime, fileref::Proof{block_id_}, + std::move(data_), std::move(P)); + } +} + +void ArchiveKeyBlockMover::skip_block_proof(td::BufferSlice data) { + data_ = std::move(data); + written_data(); +} + +void ArchiveKeyBlockMover::written_data() { + td::Ref proof_link; + if (proof_link_) { + auto p = create_proof_link(block_id_, data_.clone()).move_as_ok(); + proof_link = std::move(p); + } else { + auto p = create_proof(block_id_, data_.clone()).move_as_ok(); + proof_link = std::move(p); + } + auto ts = proof_link->get_basic_header_info().move_as_ok().utime; + auto te = ValidatorManager::persistent_state_ttl(ts); + if (te < td::Clocks::system()) { + finish_query(); + return; + } +} + +void ArchiveKeyBlockMover::abort_query(td::Status error) { + if (promise_) { + promise_.set_error(std::move(error)); + } + stop(); +} + +void ArchiveKeyBlockMover::finish_query() { + if (promise_) { + promise_.set_value(td::Unit()); + } + stop(); +} + +void ArchiveMover::start_up() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveMover::moved_blocks); + } + }); + td::actor::create_actor("fmover", masterchain_block_id_, block_db_.get(), file_db_.get(), + old_archive_db_.get(), old_archive_manager_.get(), archive_manager_.get(), + std::move(P)) + .release(); +} + +void ArchiveMover::moved_blocks() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveMover::got_handle, R.move_as_ok()); + }); + td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, masterchain_block_id_, std::move(P)); +} + +void ArchiveMover::got_handle(BlockHandle handle) { + handle_ = std::move(handle); + CHECK(handle_->is_applied()); + CHECK(handle_->inited_state_boc()); + CHECK(!handle_->deleted_state_boc()); + auto P = td::PromiseCreator::lambda( + [handle = handle_, SelfId = actor_id(this)](td::Result> R) mutable { + R.ensure(); + auto S = create_shard_state(handle->id(), R.move_as_ok()); + S.ensure(); + td::actor::send_closure(SelfId, &ArchiveMover::got_state, td::Ref{S.move_as_ok()}); + }); + td::actor::send_closure(cell_db_, &CellDb::load_cell, handle_->state(), std::move(P)); +} + +void ArchiveMover::got_state(td::Ref state) { + state_ = std::move(state); + + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks); + } + }); + + auto k = state_->prev_key_block_id(std::numeric_limits::max()); + while (k.is_valid() && k.seqno() > 0) { + td::actor::create_actor("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(), + old_archive_manager_.get(), archive_manager_.get(), ig.get_promise()) + .release(); + k = state_->prev_key_block_id(k.seqno()); + } +} + +void ArchiveMover::moved_key_blocks() { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks); + } + }); + + auto k = state_->prev_key_block_id(std::numeric_limits::max()); + while (k.is_valid() && k.seqno() > 0) { + td::actor::create_actor("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(), + old_archive_manager_.get(), archive_manager_.get(), ig.get_promise()) + .release(); + k = state_->prev_key_block_id(k.seqno()); + } +} + +void ArchiveMover::run() { + if (to_move_.empty() && to_check_.empty()) { + completed(); + return; + } + if (!to_check_.empty()) { + auto B = to_check_.back(); + CHECK(to_check_set_.count(B) == 1); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ArchiveMover::got_to_check_handle, std::move(R)); + }); + td::actor::send_closure(block_db_, &BlockDb::get_block_handle, B, std::move(P)); + return; + } + CHECK(!to_move_.empty()); +} + +void ArchiveMover::got_to_check_handle(td::Result R) { + if (R.is_error()) { + CHECK(R.error().code() == ErrorCode::notready); + run(); + return; + } + auto handle = R.move_as_ok(); +} + +} // namespace validator + +} // namespace ton diff --git a/validator/db/archive-mover.hpp b/validator/db/archive-mover.hpp new file mode 100644 index 00000000..f81ee03a --- /dev/null +++ b/validator/db/archive-mover.hpp @@ -0,0 +1,148 @@ +#pragma once + +#include "td/actor/actor.h" +#include "filedb.hpp" +#include "blockdb.hpp" +#include "statedb.hpp" +#include "celldb.hpp" +#include "archive-db.hpp" +#include "archive-manager.hpp" + +#include +#include + +namespace ton { + +namespace validator { + +class ArchiveFileMover : public td::actor::Actor { + public: + ArchiveFileMover(BlockIdExt block_id, td::actor::ActorId block_db, td::actor::ActorId file_db, + td::actor::ActorId old_archive_db, td::actor::ActorId old_archive_manager, + td::actor::ActorId archive_manager, td::Promise promise) + : block_id_(block_id) + , block_db_(block_db) + , file_db_(file_db) + , old_archive_db_(old_archive_db) + , old_archive_manager_(old_archive_manager) + , archive_manager_(archive_manager) + , promise_(std::move(promise)) { + } + void start_up() override; + void got_block_handle0(td::Result R); + void got_block_handle1(td::Result R); + void got_block_handle2(td::Result R); + void got_block_handle(); + + void processed_child(); + void processed_all_children(); + + void got_block_data(td::Result R); + void got_block_proof(td::Result R); + void got_block_proof_link(td::Result R); + + void written_data(); + void written_handle(); + + void abort_query(td::Status error); + void finish_query(); + + private: + BlockIdExt block_id_; + BlockHandle handle_; + td::BufferSlice data_; + td::BufferSlice proof_; + td::BufferSlice proof_link_; + bool left_ = true; + + td::actor::ActorId block_db_; + td::actor::ActorId file_db_; + td::actor::ActorId old_archive_db_; + td::actor::ActorId old_archive_manager_; + td::actor::ActorId archive_manager_; + + td::Promise promise_; +}; + +class ArchiveKeyBlockMover : public td::actor::Actor { + public: + ArchiveKeyBlockMover(BlockIdExt block_id, td::actor::ActorId block_db, td::actor::ActorId file_db, + td::actor::ActorId old_archive_db, + td::actor::ActorId old_archive_manager, + td::actor::ActorId archive_manager, td::Promise promise) + : block_id_(block_id) + , block_db_(block_db) + , file_db_(file_db) + , old_archive_db_(old_archive_db) + , old_archive_manager_(old_archive_manager) + , archive_manager_(archive_manager) + , promise_(std::move(promise)) { + } + + void start_up() override; + void failed_to_get_proof0(); + void failed_to_get_proof1(); + void failed_to_get_proof2(); + void failed_to_get_proof3(); + void got_block_proof(td::BufferSlice data); + void skip_block_proof(td::BufferSlice data); + + void written_data(); + + void abort_query(td::Status error); + void finish_query(); + + private: + BlockIdExt block_id_; + td::BufferSlice data_; + bool proof_link_ = false; + + td::actor::ActorId block_db_; + td::actor::ActorId file_db_; + td::actor::ActorId old_archive_db_; + td::actor::ActorId old_archive_manager_; + td::actor::ActorId archive_manager_; + + td::Promise promise_; +}; + +class ArchiveMover : public td::actor::Actor { + public: + ArchiveMover(std::string db_root, BlockIdExt masterchain_block_id, BlockIdExt shard_block_id, + BlockIdExt key_block_id); + + void start_up() override; + void moved_blocks(); + void got_handle(BlockHandle handle); + void got_state(td::Ref state); + void moved_key_blocks(); + void run(); + void completed(); + void add_to_move(BlockIdExt block_id); + void add_to_check(BlockIdExt block_id); + + void got_to_check_handle(td::Result R); + + void abort_query(td::Status error); + void finish_query(); + + private: + std::string db_root_; + BlockHandle handle_; + td::Ref state_; + + td::actor::ActorOwn block_db_; + td::actor::ActorOwn file_db_; + td::actor::ActorOwn old_archive_db_; + td::actor::ActorOwn old_archive_manager_; + td::actor::ActorOwn archive_manager_; + td::actor::ActorOwn cell_db_; + + BlockIdExt masterchain_block_id_; + BlockIdExt shard_block_id_; + BlockIdExt key_block_id_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp new file mode 100644 index 00000000..b623415a --- /dev/null +++ b/validator/db/archive-slice.cpp @@ -0,0 +1,470 @@ +#include "archive-slice.hpp" + +#include "td/actor/MultiPromise.h" +#include "validator/fabric.h" +#include "td/db/RocksDb.h" +#include "ton/ton-io.hpp" +#include "td/utils/port/path.h" +#include "common/delay.h" +#include "files-async.hpp" + +namespace ton { + +namespace validator { + +void PackageWriter::append(std::string filename, td::BufferSlice data, + td::Promise> promise) { + auto offset = package_->append(std::move(filename), std::move(data), !async_mode_); + auto size = package_->size(); + + promise.set_value(std::pair{offset, size}); +} + +class PackageReader : public td::actor::Actor { + public: + PackageReader(std::shared_ptr package, td::uint64 offset, + td::Promise> promise) + : package_(std::move(package)), offset_(offset), promise_(std::move(promise)) { + } + void start_up() { + promise_.set_result(package_->read(offset_)); + } + + private: + std::shared_ptr package_; + td::uint64 offset_; + td::Promise> promise_; +}; + +void ArchiveSlice::add_handle(BlockHandle handle, td::Promise promise) { + if (destroyed_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); + return; + } + if (handle->id().seqno() == 0) { + update_handle(std::move(handle), std::move(promise)); + return; + } + CHECK(!key_blocks_only_); + CHECK(!temp_); + CHECK(handle->inited_unix_time()); + CHECK(handle->inited_logical_time()); + + auto key = get_db_key_lt_desc(handle->id().shard_full()); + + std::string value; + auto R = kv_->get(key.as_slice(), value); + R.ensure(); + tl_object_ptr v; + bool add_shard = false; + if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(td::BufferSlice{value}, true); + F.ensure(); + v = F.move_as_ok(); + } else { + v = create_tl_object(1, 1, 0, 0, 0); + add_shard = true; + } + if (handle->id().seqno() <= static_cast(v->last_seqno_) || + handle->logical_time() <= static_cast(v->last_lt_) || + handle->unix_time() <= static_cast(v->last_ts_)) { + update_handle(std::move(handle), std::move(promise)); + return; + } + auto db_value = create_serialize_tl_object(create_tl_block_id(handle->id()), + handle->logical_time(), handle->unix_time()); + auto db_key = get_db_key_lt_el(handle->id().shard_full(), v->last_idx_++); + auto status_key = create_serialize_tl_object(); + v->last_seqno_ = handle->id().seqno(); + v->last_lt_ = handle->logical_time(); + v->last_ts_ = handle->unix_time(); + + td::uint32 idx = 0; + if (add_shard) { + auto G = kv_->get(status_key.as_slice(), value); + G.ensure(); + if (G.move_as_ok() == td::KeyValue::GetStatus::NotFound) { + idx = 0; + } else { + auto F = fetch_tl_object(value, true); + F.ensure(); + auto f = F.move_as_ok(); + idx = f->total_shards_; + } + } + + auto version = handle->version(); + + begin_transaction(); + kv_->set(key, serialize_tl_object(v, true)).ensure(); + kv_->set(db_key, db_value.as_slice()).ensure(); + if (add_shard) { + auto shard_key = create_serialize_tl_object(idx); + auto shard_value = + create_serialize_tl_object(handle->id().id.workchain, handle->id().id.shard); + kv_->set(status_key.as_slice(), create_serialize_tl_object(idx + 1)).ensure(); + kv_->set(shard_key.as_slice(), shard_value.as_slice()).ensure(); + } + kv_->set(get_db_key_block_info(handle->id()), handle->serialize().as_slice()).ensure(); + commit_transaction(); + + handle->flushed_upto(version); + handle->set_handle_moved_to_archive(); + + if (handle->need_flush()) { + update_handle(std::move(handle), std::move(promise)); + } else { + promise.set_value(td::Unit()); + } +} + +void ArchiveSlice::update_handle(BlockHandle handle, td::Promise promise) { + if (destroyed_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); + return; + } + if (!handle->need_flush() && (temp_ || handle->handle_moved_to_archive())) { + promise.set_value(td::Unit()); + return; + } + CHECK(!key_blocks_only_); + + begin_transaction(); + do { + auto version = handle->version(); + kv_->set(get_db_key_block_info(handle->id()), handle->serialize().as_slice()).ensure(); + handle->flushed_upto(version); + } while (handle->need_flush()); + commit_transaction(); + if (!temp_) { + handle->set_handle_moved_to_archive(); + } + + promise.set_value(td::Unit()); +} + +void ArchiveSlice::add_file(FileReference ref_id, td::BufferSlice data, td::Promise promise) { + if (destroyed_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); + return; + } + std::string value; + auto R = kv_->get(ref_id.hash().to_hex(), value); + R.ensure(); + if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) { + promise.set_value(td::Unit()); + return; + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), ref_id, promise = std::move(promise)]( + td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto v = R.move_as_ok(); + td::actor::send_closure(SelfId, &ArchiveSlice::add_file_cont, std::move(ref_id), v.first, v.second, + std::move(promise)); + }); + + td::actor::send_closure(writer_, &PackageWriter::append, ref_id.filename(), std::move(data), std::move(P)); +} + +void ArchiveSlice::add_file_cont(FileReference ref_id, td::uint64 offset, td::uint64 size, + td::Promise promise) { + if (destroyed_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); + return; + } + begin_transaction(); + kv_->set("status", td::to_string(size)).ensure(); + kv_->set(ref_id.hash().to_hex(), td::to_string(offset)).ensure(); + commit_transaction(); + promise.set_value(td::Unit()); +} + +void ArchiveSlice::get_handle(BlockIdExt block_id, td::Promise promise) { + if (destroyed_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); + return; + } + CHECK(!key_blocks_only_); + std::string value; + auto R = kv_->get(get_db_key_block_info(block_id), value); + R.ensure(); + if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) { + promise.set_error(td::Status::Error(ErrorCode::notready, "handle not in archive slice")); + return; + } + auto E = create_block_handle(td::BufferSlice{value}); + E.ensure(); + auto handle = E.move_as_ok(); + if (!temp_) { + handle->set_handle_moved_to_archive(); + } + promise.set_value(std::move(handle)); +} + +void ArchiveSlice::get_file(FileReference ref_id, td::Promise promise) { + if (destroyed_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); + return; + } + std::string value; + auto R = kv_->get(ref_id.hash().to_hex(), value); + R.ensure(); + if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) { + promise.set_error(td::Status::Error(ErrorCode::notready, "file not in archive slice")); + return; + } + auto offset = td::to_integer(value); + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + promise.set_value(std::move(R.move_as_ok().second)); + } + }); + td::actor::create_actor("reader", package_, offset, std::move(P)).release(); +} + +void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id, + std::function compare_desc, + std::function compare, bool exact, + td::Promise promise) { + if (destroyed_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); + return; + } + bool f = false; + BlockIdExt block_id; + td::uint32 ls = 0; + for (td::uint32 len = 0; len <= 60; len++) { + auto s = shard_prefix(account_id, len); + auto key = get_db_key_lt_desc(s); + std::string value; + auto F = kv_->get(key, value); + F.ensure(); + if (F.move_as_ok() == td::KeyValue::GetStatus::NotFound) { + if (!f) { + continue; + } else { + break; + } + } + f = true; + auto G = fetch_tl_object(td::BufferSlice{value}, true); + G.ensure(); + auto g = G.move_as_ok(); + if (compare_desc(*g.get()) > 0) { + continue; + } + td::uint32 l = g->first_idx_ - 1; + BlockIdExt lseq; + td::uint32 r = g->last_idx_; + BlockIdExt rseq; + while (r - l > 1) { + auto x = (r + l) / 2; + auto db_key = get_db_key_lt_el(s, x); + F = kv_->get(db_key, value); + F.ensure(); + CHECK(F.move_as_ok() == td::KeyValue::GetStatus::Ok); + auto E = fetch_tl_object(td::BufferSlice{value}, true); + E.ensure(); + auto e = E.move_as_ok(); + int cmp_val = compare(*e.get()); + + if (cmp_val < 0) { + rseq = create_block_id(e->id_); + r = x; + } else if (cmp_val > 0) { + lseq = create_block_id(e->id_); + l = x; + } else { + get_handle(create_block_id(e->id_), std::move(promise)); + return; + } + } + if (rseq.is_valid()) { + if (!block_id.is_valid()) { + block_id = rseq; + } else if (block_id.id.seqno > rseq.id.seqno) { + block_id = rseq; + } + } + if (lseq.is_valid()) { + if (ls < lseq.id.seqno) { + ls = lseq.id.seqno; + } + } + if (block_id.is_valid() && ls + 1 == block_id.id.seqno) { + if (!exact) { + get_handle(block_id, std::move(promise)); + } else { + promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found")); + } + return; + } + } + if (!exact && block_id.is_valid()) { + get_handle(block_id, std::move(promise)); + } else { + promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found")); + } +} + +void ArchiveSlice::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise) { + return get_block_common( + account_id, + [lt](ton_api::db_lt_desc_value &w) { + return lt > static_cast(w.last_lt_) ? 1 : lt == static_cast(w.last_lt_) ? 0 : -1; + }, + [lt](ton_api::db_lt_el_value &w) { + return lt > static_cast(w.lt_) ? 1 : lt == static_cast(w.lt_) ? 0 : -1; + }, + false, std::move(promise)); +} + +void ArchiveSlice::get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, + td::Promise promise) { + return get_block_common( + account_id, + [seqno](ton_api::db_lt_desc_value &w) { + return seqno > static_cast(w.last_seqno_) + ? 1 + : seqno == static_cast(w.last_seqno_) ? 0 : -1; + }, + [seqno](ton_api::db_lt_el_value &w) { + return seqno > static_cast(w.id_->seqno_) + ? 1 + : seqno == static_cast(w.id_->seqno_) ? 0 : -1; + }, + true, std::move(promise)); +} + +void ArchiveSlice::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, + td::Promise promise) { + return get_block_common( + account_id, + [ts](ton_api::db_lt_desc_value &w) { + return ts > static_cast(w.last_ts_) ? 1 : ts == static_cast(w.last_ts_) ? 0 : -1; + }, + [ts](ton_api::db_lt_el_value &w) { + return ts > static_cast(w.ts_) ? 1 : ts == static_cast(w.ts_) ? 0 : -1; + }, + false, std::move(promise)); +} + +td::BufferSlice ArchiveSlice::get_db_key_lt_desc(ShardIdFull shard) { + return create_serialize_tl_object(shard.workchain, shard.shard); +} + +td::BufferSlice ArchiveSlice::get_db_key_lt_el(ShardIdFull shard, td::uint32 idx) { + return create_serialize_tl_object(shard.workchain, shard.shard, idx); +} + +td::BufferSlice ArchiveSlice::get_db_key_block_info(BlockIdExt block_id) { + return create_serialize_tl_object(create_tl_block_id(block_id)); +} + +void ArchiveSlice::get_slice(td::uint64 offset, td::uint32 limit, td::Promise promise) { + td::actor::create_actor("readfile", prefix_ + ".pack", offset, limit, 0, std::move(promise)).release(); +} + +void ArchiveSlice::start_up() { + auto R = Package::open(prefix_ + ".pack", false, true); + if (R.is_error()) { + LOG(FATAL) << "failed to open/create archive '" << prefix_ << ".pack" + << "': " << R.move_as_error(); + return; + } + package_ = std::make_shared(R.move_as_ok()); + kv_ = std::make_shared(td::RocksDb::open(prefix_ + ".index").move_as_ok()); + + std::string value; + auto R2 = kv_->get("status", value); + R2.ensure(); + + if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { + auto len = td::to_integer(value); + package_->truncate(len); + } else { + package_->truncate(0); + } + + writer_ = td::actor::create_actor("writer", package_); +} + +void ArchiveSlice::begin_transaction() { + if (!async_mode_ || !huge_transaction_started_) { + kv_->begin_transaction().ensure(); + if (async_mode_) { + huge_transaction_started_ = true; + } + } +} + +void ArchiveSlice::commit_transaction() { + if (!async_mode_ || huge_transaction_size_++ >= 100) { + kv_->commit_transaction().ensure(); + if (async_mode_) { + huge_transaction_size_ = 0; + huge_transaction_started_ = false; + } + } +} + +void ArchiveSlice::set_async_mode(bool mode, td::Promise promise) { + async_mode_ = mode; + if (!async_mode_ && huge_transaction_started_) { + kv_->commit_transaction().ensure(); + huge_transaction_size_ = 0; + huge_transaction_started_ = false; + } + + td::actor::send_closure(writer_, &PackageWriter::set_async_mode, mode, std::move(promise)); +} + +ArchiveSlice::ArchiveSlice(bool key_blocks_only, bool temp, std::string prefix) + : key_blocks_only_(key_blocks_only), temp_(temp), prefix_(std::move(prefix)) { +} + +namespace { + +void destroy_db(std::string name, td::uint32 attempt, td::Promise promise) { + auto S = td::RocksDb::destroy(name); + if (S.is_ok()) { + promise.set_value(td::Unit()); + return; + } + if (S.is_error() && attempt > 0 && attempt % 64 == 0) { + LOG(ERROR) << "failed to destroy index " << name << ": " << S; + } else { + LOG(DEBUG) << "failed to destroy index " << name << ": " << S; + } + delay_action( + [name, attempt, promise = std::move(promise)]() mutable { destroy_db(name, attempt, std::move(promise)); }, + td::Timestamp::in(1.0)); +} +} // namespace + +void ArchiveSlice::destroy(td::Promise promise) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + destroyed_ = true; + + writer_.reset(); + package_ = nullptr; + kv_ = nullptr; + + td::unlink(prefix_ + ".pack").ensure(); + + delay_action([name = prefix_ + ".index", attempt = 0, + promise = ig.get_promise()]() mutable { destroy_db(name, attempt, std::move(promise)); }, + td::Timestamp::in(0.0)); +} + +} // namespace validator + +} // namespace ton diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp new file mode 100644 index 00000000..b4e03e01 --- /dev/null +++ b/validator/db/archive-slice.hpp @@ -0,0 +1,84 @@ +#pragma once + +#include "validator/interfaces/db.h" +#include "package.hpp" +#include "fileref.hpp" + +namespace ton { + +namespace validator { + +class PackageWriter : public td::actor::Actor { + public: + PackageWriter(std::shared_ptr package) : package_(std::move(package)) { + } + + void append(std::string filename, td::BufferSlice data, td::Promise> promise); + void set_async_mode(bool mode, td::Promise promise) { + async_mode_ = mode; + if (!async_mode_) { + package_->sync(); + } + promise.set_value(td::Unit()); + } + + private: + std::shared_ptr package_; + bool async_mode_ = false; +}; + +class ArchiveSlice : public td::actor::Actor { + public: + ArchiveSlice(bool key_blocks_only, bool temp, std::string prefix); + + void add_handle(BlockHandle handle, td::Promise promise); + void update_handle(BlockHandle handle, td::Promise promise); + void add_file(FileReference ref_id, td::BufferSlice data, td::Promise promise); + void get_handle(BlockIdExt block_id, td::Promise promise); + void get_file(FileReference ref_id, td::Promise promise); + + /* from LTDB */ + void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise); + void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise); + void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise promise); + void get_block_common(AccountIdPrefixFull account_id, + std::function compare_desc, + std::function compare, bool exact, + td::Promise promise); + + void get_slice(td::uint64 offset, td::uint32 limit, td::Promise promise); + + void start_up() override; + void destroy(td::Promise promise); + + void begin_transaction(); + void commit_transaction(); + void set_async_mode(bool mode, td::Promise promise); + + private: + void written_data(BlockHandle handle, td::Promise promise); + void add_file_cont(FileReference ref_id, td::uint64 offset, td::uint64 size, td::Promise promise); + + /* ltdb */ + td::BufferSlice get_db_key_lt_desc(ShardIdFull shard); + td::BufferSlice get_db_key_lt_el(ShardIdFull shard, td::uint32 idx); + td::BufferSlice get_db_key_block_info(BlockIdExt block_id); + td::BufferSlice get_lt_from_db(ShardIdFull shard, td::uint32 idx); + + bool key_blocks_only_; + bool temp_; + + bool destroyed_ = false; + bool async_mode_ = false; + bool huge_transaction_started_ = false; + td::uint32 huge_transaction_size_ = 0; + + std::string prefix_; + std::shared_ptr package_; + std::shared_ptr kv_; + td::actor::ActorOwn writer_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/db/archiver.cpp b/validator/db/archiver.cpp index f0f1692c..5b0d1a8a 100644 --- a/validator/db/archiver.cpp +++ b/validator/db/archiver.cpp @@ -24,41 +24,30 @@ namespace ton { namespace validator { -BlockArchiver::BlockArchiver(BlockIdExt block_id, td::actor::ActorId root_db, - td::actor::ActorId file_db, td::actor::ActorId archive_db, - td::actor::ActorId archive, td::Promise promise) - : block_id_(block_id) - , root_db_(root_db) - , file_db_(file_db) - , archive_db_(archive_db) - , archive_(archive) - , promise_(std::move(promise)) { +BlockArchiver::BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, + td::Promise promise) + : handle_(std::move(handle)), archive_(archive_db), promise_(std::move(promise)) { } void BlockArchiver::start_up() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &BlockArchiver::got_block_handle, R.move_as_ok()); - }); - td::actor::send_closure(root_db_, &RootDb::get_block_handle_external, block_id_, false, std::move(P)); + if (handle_->handle_moved_to_archive()) { + moved_handle(); + } else { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &BlockArchiver::moved_handle); + }); + td::actor::send_closure(archive_, &ArchiveManager::add_handle, handle_, std::move(P)); + } } -void BlockArchiver::got_block_handle(BlockHandle handle) { - handle_ = std::move(handle); +void BlockArchiver::moved_handle() { + CHECK(handle_->handle_moved_to_archive()); if (handle_->moved_to_archive()) { finish_query(); return; } - if (!handle_->is_applied() && !handle_->is_archived() && - (!handle_->inited_is_key_block() || !handle_->is_key_block())) { - // no need for this block - // probably this block not in final chain - // this will eventually delete all associated data - written_block_data(); - return; - } - if (!handle_->inited_proof()) { written_proof(); return; @@ -69,11 +58,7 @@ void BlockArchiver::got_block_handle(BlockHandle handle) { td::actor::send_closure(SelfId, &BlockArchiver::got_proof, R.move_as_ok()); }); - if (handle_->moved_to_storage()) { - td::actor::send_closure(archive_db_, &FileDb::load_file, FileDb::RefId{fileref::Proof{block_id_}}, std::move(P)); - } else { - td::actor::send_closure(file_db_, &FileDb::load_file, FileDb::RefId{fileref::Proof{block_id_}}, std::move(P)); - } + td::actor::send_closure(archive_, &ArchiveManager::get_temp_file_short, fileref::Proof{handle_->id()}, std::move(P)); } void BlockArchiver::got_proof(td::BufferSlice data) { @@ -81,8 +66,8 @@ void BlockArchiver::got_proof(td::BufferSlice data) { R.ensure(); td::actor::send_closure(SelfId, &BlockArchiver::written_proof); }); - td::actor::send_closure(archive_, &ArchiveManager::write, handle_->unix_time(), handle_->is_key_block(), - FileDb::RefId{fileref::Proof{block_id_}}, std::move(data), std::move(P)); + td::actor::send_closure(archive_, &ArchiveManager::add_file, handle_, fileref::Proof{handle_->id()}, std::move(data), + std::move(P)); } void BlockArchiver::written_proof() { @@ -95,12 +80,9 @@ void BlockArchiver::written_proof() { R.ensure(); td::actor::send_closure(SelfId, &BlockArchiver::got_proof_link, R.move_as_ok()); }); - if (handle_->moved_to_storage()) { - td::actor::send_closure(archive_db_, &FileDb::load_file, FileDb::RefId{fileref::ProofLink{block_id_}}, - std::move(P)); - } else { - td::actor::send_closure(file_db_, &FileDb::load_file, FileDb::RefId{fileref::ProofLink{block_id_}}, std::move(P)); - } + + td::actor::send_closure(archive_, &ArchiveManager::get_temp_file_short, fileref::ProofLink{handle_->id()}, + std::move(P)); } void BlockArchiver::got_proof_link(td::BufferSlice data) { @@ -108,8 +90,8 @@ void BlockArchiver::got_proof_link(td::BufferSlice data) { R.ensure(); td::actor::send_closure(SelfId, &BlockArchiver::written_proof_link); }); - td::actor::send_closure(archive_, &ArchiveManager::write, handle_->unix_time(), handle_->is_key_block(), - FileDb::RefId{fileref::ProofLink{block_id_}}, std::move(data), std::move(P)); + td::actor::send_closure(archive_, &ArchiveManager::add_file, handle_, fileref::ProofLink{handle_->id()}, + std::move(data), std::move(P)); } void BlockArchiver::written_proof_link() { @@ -121,11 +103,8 @@ void BlockArchiver::written_proof_link() { R.ensure(); td::actor::send_closure(SelfId, &BlockArchiver::got_block_data, R.move_as_ok()); }); - if (handle_->moved_to_storage()) { - td::actor::send_closure(archive_db_, &FileDb::load_file, FileDb::RefId{fileref::Block{block_id_}}, std::move(P)); - } else { - td::actor::send_closure(file_db_, &FileDb::load_file, FileDb::RefId{fileref::Block{block_id_}}, std::move(P)); - } + + td::actor::send_closure(archive_, &ArchiveManager::get_temp_file_short, fileref::Block{handle_->id()}, std::move(P)); } void BlockArchiver::got_block_data(td::BufferSlice data) { @@ -133,8 +112,8 @@ void BlockArchiver::got_block_data(td::BufferSlice data) { R.ensure(); td::actor::send_closure(SelfId, &BlockArchiver::written_block_data); }); - td::actor::send_closure(archive_, &ArchiveManager::write, handle_->unix_time(), handle_->is_key_block(), - FileDb::RefId{fileref::Block{block_id_}}, std::move(data), std::move(P)); + td::actor::send_closure(archive_, &ArchiveManager::add_file, handle_, fileref::Block{handle_->id()}, std::move(data), + std::move(P)); } void BlockArchiver::written_block_data() { @@ -144,7 +123,7 @@ void BlockArchiver::written_block_data() { R.ensure(); td::actor::send_closure(SelfId, &BlockArchiver::finish_query); }); - td::actor::send_closure(root_db_, &RootDb::store_block_handle, handle_, std::move(P)); + td::actor::send_closure(archive_, &ArchiveManager::update_handle, handle_, std::move(P)); } void BlockArchiver::finish_query() { @@ -156,7 +135,7 @@ void BlockArchiver::finish_query() { void BlockArchiver::abort_query(td::Status reason) { if (promise_) { - VLOG(VALIDATOR_WARNING) << "failed to archive block " << block_id_ << ": " << reason; + VLOG(VALIDATOR_WARNING) << "failed to archive block " << handle_->id() << ": " << reason; promise_.set_error(std::move(reason)); } stop(); diff --git a/validator/db/archiver.hpp b/validator/db/archiver.hpp index 3335d3aa..054ab42b 100644 --- a/validator/db/archiver.hpp +++ b/validator/db/archiver.hpp @@ -22,7 +22,7 @@ #include "td/actor/actor.h" #include "validator/interfaces/block-handle.h" #include "ton/ton-io.hpp" -#include "archive-db.hpp" +#include "archive-manager.hpp" namespace ton { @@ -33,14 +33,12 @@ class FileDb; class BlockArchiver : public td::actor::Actor { public: - BlockArchiver(BlockIdExt block_id, td::actor::ActorId root_db, td::actor::ActorId file_db, - td::actor::ActorId archive_db, td::actor::ActorId archive, - td::Promise promise); + BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, td::Promise promise); void abort_query(td::Status error); void start_up() override; - void got_block_handle(BlockHandle handle); + void moved_handle(); void got_proof(td::BufferSlice data); void written_proof(); void got_proof_link(td::BufferSlice data); @@ -50,14 +48,9 @@ class BlockArchiver : public td::actor::Actor { void finish_query(); private: - BlockIdExt block_id_; - td::actor::ActorId root_db_; - td::actor::ActorId file_db_; - td::actor::ActorId archive_db_; + BlockHandle handle_; td::actor::ActorId archive_; td::Promise promise_; - - BlockHandle handle_; }; } // namespace validator diff --git a/validator/db/blockdb.cpp b/validator/db/blockdb.cpp deleted file mode 100644 index 1e1c0cf0..00000000 --- a/validator/db/blockdb.cpp +++ /dev/null @@ -1,301 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#include "blockdb.hpp" -#include "rootdb.hpp" -#include "validator/fabric.h" -#include "ton/ton-tl.hpp" -#include "td/utils/port/path.h" -#include "files-async.hpp" -#include "td/db/RocksDb.h" - -namespace ton { - -namespace validator { - -void BlockDb::store_block_handle(BlockHandle handle, td::Promise promise) { - if (!handle->id().is_valid()) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid block id")); - return; - } - auto lru_key = get_block_lru_key(handle->id()); - auto value_key = get_block_value_key(handle->id()); - while (handle->need_flush()) { - auto version = handle->version(); - auto R = get_block_value(value_key); - if (R.is_ok()) { - kv_->begin_transaction().ensure(); - set_block_value(value_key, handle->serialize()); - kv_->commit_transaction().ensure(); - } else { - CHECK(get_block_lru(lru_key).is_error()); - auto empty = get_block_lru_empty_key_hash(); - auto ER = get_block_lru(empty); - ER.ensure(); - auto E = ER.move_as_ok(); - - auto PR = get_block_lru(E.prev); - PR.ensure(); - auto P = PR.move_as_ok(); - CHECK(P.next == empty); - - DbEntry D{handle->id(), E.prev, empty}; - - E.prev = lru_key; - P.next = lru_key; - - if (P.is_empty()) { - E.next = lru_key; - P.prev = lru_key; - } - - kv_->begin_transaction().ensure(); - set_block_value(value_key, handle->serialize()); - set_block_lru(empty, std::move(E)); - set_block_lru(D.prev, std::move(P)); - set_block_lru(lru_key, std::move(D)); - kv_->commit_transaction().ensure(); - } - handle->flushed_upto(version); - } - promise.set_value(td::Unit()); -} - -void BlockDb::get_block_handle(BlockIdExt id, td::Promise promise) { - if (!id.is_valid()) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid block id")); - return; - } - CHECK(id.is_valid()); - auto key_hash = get_block_value_key(id); - auto B = get_block_value(key_hash); - if (B.is_error()) { - promise.set_error(B.move_as_error()); - return; - } - promise.set_result(create_block_handle(B.move_as_ok())); -} - -void BlockDb::start_up() { - kv_ = std::make_shared(td::RocksDb::open(db_path_).move_as_ok()); - alarm_timestamp() = td::Timestamp::in(0.1); - - auto empty = get_block_lru_empty_key_hash(); - if (get_block_lru(empty).is_error()) { - DbEntry e{BlockIdExt{}, empty, empty}; - kv_->begin_transaction().ensure(); - set_block_lru(empty, std::move(e)); - kv_->commit_transaction().ensure(); - } - last_gc_ = empty; -} - -BlockDb::BlockDb(td::actor::ActorId root_db, std::string db_path) : root_db_(root_db), db_path_(db_path) { -} - -BlockDb::KeyHash BlockDb::get_block_lru_key(BlockIdExt id) { - if (!id.is_valid()) { - return KeyHash::zero(); - } else { - auto obj = create_tl_object(create_tl_block_id(id)); - return get_tl_object_sha_bits256(obj); - } -} - -BlockDb::KeyHash BlockDb::get_block_value_key(BlockIdExt id) { - CHECK(id.is_valid()); - auto obj = create_tl_object(create_tl_block_id(id)); - return get_tl_object_sha_bits256(obj); -} - -td::Result BlockDb::get_block_lru(KeyHash key_hash) { - std::string value; - auto R = kv_->get(key_hash.as_slice(), value); - R.ensure(); - if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) { - return td::Status::Error(ErrorCode::notready, "not in db"); - } - auto v = fetch_tl_object(td::BufferSlice{value}, true); - v.ensure(); - return DbEntry{v.move_as_ok()}; -} - -td::Result BlockDb::get_block_value(KeyHash key_hash) { - std::string value; - auto R = kv_->get(key_hash.as_slice(), value); - R.ensure(); - if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) { - return td::Status::Error(ErrorCode::notready, "not in db"); - } - return td::BufferSlice{value}; -} - -void BlockDb::set_block_lru(KeyHash key_hash, DbEntry e) { - kv_->set(key_hash.as_slice(), serialize_tl_object(e.release(), true)).ensure(); -} - -void BlockDb::set_block_value(KeyHash key_hash, td::BufferSlice value) { - kv_->set(key_hash.as_slice(), std::move(value)).ensure(); -} - -void BlockDb::alarm() { - auto R = get_block_lru(last_gc_); - R.ensure(); - - auto N = R.move_as_ok(); - if (N.is_empty()) { - last_gc_ = N.next; - alarm_timestamp() = td::Timestamp::in(0.01); - return; - } - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &BlockDb::skip_gc); - } else { - auto value = R.move_as_ok(); - if (!value) { - td::actor::send_closure(SelfId, &BlockDb::skip_gc); - } else { - td::actor::send_closure(SelfId, &BlockDb::gc); - } - } - }); - td::actor::send_closure(root_db_, &RootDb::allow_block_gc, N.block_id, std::move(P)); -} - -void BlockDb::gc() { - auto FR = get_block_lru(last_gc_); - FR.ensure(); - auto F = FR.move_as_ok(); - - auto PR = get_block_lru(F.prev); - PR.ensure(); - auto P = PR.move_as_ok(); - auto NR = get_block_lru(F.next); - NR.ensure(); - auto N = NR.move_as_ok(); - - P.next = F.next; - N.prev = F.prev; - if (P.is_empty() && N.is_empty()) { - P.prev = P.next; - N.next = N.prev; - } - - auto value_key = get_block_value_key(F.block_id); - - kv_->begin_transaction().ensure(); - kv_->erase(last_gc_.as_slice()).ensure(); - kv_->erase(value_key.as_slice()).ensure(); - set_block_lru(F.prev, std::move(P)); - set_block_lru(F.next, std::move(N)); - kv_->commit_transaction().ensure(); - alarm_timestamp() = td::Timestamp::now(); - - DCHECK(get_block_lru(last_gc_).is_error()); - last_gc_ = F.next; -} - -void BlockDb::skip_gc() { - auto R = get_block_lru(last_gc_); - R.ensure(); - - auto N = R.move_as_ok(); - last_gc_ = N.next; - alarm_timestamp() = td::Timestamp::in(0.01); -} - -BlockDb::DbEntry::DbEntry(tl_object_ptr entry) - : block_id(create_block_id(entry->id_)), prev(entry->prev_), next(entry->next_) { -} - -tl_object_ptr BlockDb::DbEntry::release() { - return create_tl_object(create_tl_block_id(block_id), prev, next); -} - -void BlockDb::truncate(td::Ref state, td::Promise promise) { - std::map max_seqno; - max_seqno.emplace(ShardIdFull{masterchainId}, state->get_seqno() + 1); - - auto shards = state->get_shards(); - auto it = KeyHash::zero(); - kv_->begin_transaction().ensure(); - while (true) { - auto R = get_block_lru(it); - R.ensure(); - auto v = R.move_as_ok(); - it = v.next; - R = get_block_lru(it); - R.ensure(); - v = R.move_as_ok(); - if (v.is_empty()) { - break; - } - - auto s = v.block_id.shard_full(); - if (!max_seqno.count(s)) { - bool found = false; - for (auto &shard : shards) { - if (shard_intersects(shard->shard(), s)) { - found = true; - max_seqno.emplace(s, shard->top_block_id().seqno() + 1); - break; - } - } - if (!found) { - max_seqno.emplace(s, 0); - } - } - - bool to_delete = v.block_id.seqno() >= max_seqno[s]; - if (to_delete) { - auto key_hash = get_block_value_key(v.block_id); - auto B = get_block_value(key_hash); - B.ensure(); - auto handleR = create_block_handle(B.move_as_ok()); - handleR.ensure(); - auto handle = handleR.move_as_ok(); - - handle->unsafe_clear_applied(); - handle->unsafe_clear_next(); - - if (handle->need_flush()) { - set_block_value(key_hash, handle->serialize()); - } - } else if (v.block_id.seqno() + 1 == max_seqno[s]) { - auto key_hash = get_block_value_key(v.block_id); - auto B = get_block_value(key_hash); - B.ensure(); - auto handleR = create_block_handle(B.move_as_ok()); - handleR.ensure(); - auto handle = handleR.move_as_ok(); - - handle->unsafe_clear_next(); - - if (handle->need_flush()) { - set_block_value(key_hash, handle->serialize()); - } - } - } - kv_->commit_transaction().ensure(); -} - -} // namespace validator - -} // namespace ton diff --git a/validator/db/blockdb.hpp b/validator/db/blockdb.hpp deleted file mode 100644 index e9e26c8c..00000000 --- a/validator/db/blockdb.hpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#pragma once - -#include "td/actor/actor.h" -#include "ton/ton-types.h" -#include "validator/interfaces/block-handle.h" -#include "validator/interfaces/db.h" -#include "td/db/KeyValueAsync.h" - -namespace ton { - -namespace validator { - -class RootDb; - -class BlockDb : public td::actor::Actor { - public: - void store_block_handle(BlockHandle handle, td::Promise promise); - void get_block_handle(BlockIdExt id, td::Promise promise); - - void start_up() override; - void alarm() override; - - void gc(); - void skip_gc(); - - void truncate(td::Ref state, td::Promise promise); - - BlockDb(td::actor::ActorId root_db, std::string db_path); - - private: - using KeyHash = td::Bits256; - struct DbEntry { - BlockIdExt block_id; - KeyHash prev; - KeyHash next; - - DbEntry(tl_object_ptr entry); - DbEntry() { - } - DbEntry(BlockIdExt block_id, KeyHash prev, KeyHash next) : block_id(block_id), prev(prev), next(next) { - } - tl_object_ptr release(); - bool is_empty() const { - return !block_id.is_valid(); - } - }; - static KeyHash get_block_lru_key(BlockIdExt block_id); - static KeyHash get_block_value_key(BlockIdExt block_id); - static KeyHash get_block_lru_empty_key_hash() { - return KeyHash::zero(); - } - - td::Result get_block_lru(KeyHash key); - td::Result get_block_value(KeyHash key); - - void set_block_lru(KeyHash key_hash, DbEntry e); - void set_block_value(KeyHash key_hash, td::BufferSlice data); - - std::shared_ptr kv_; - - td::actor::ActorId root_db_; - - std::string db_path_; - - KeyHash last_gc_ = KeyHash::zero(); -}; - -} // namespace validator - -} // namespace ton diff --git a/validator/db/filedb.cpp b/validator/db/filedb.cpp deleted file mode 100644 index 371c08fc..00000000 --- a/validator/db/filedb.cpp +++ /dev/null @@ -1,400 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#include "filedb.hpp" -#include "rootdb.hpp" -#include "td/utils/port/path.h" -#include "files-async.hpp" -#include "adnl/utils.hpp" -#include "tl-utils/tl-utils.hpp" -#include "td/utils/overloaded.h" - -#include "td/db/RocksDb.h" - -namespace ton { - -namespace validator { - -std::string FileDb::get_file_name(const RefId& ref_id, bool create_dirs) { - auto ref_id_hash = get_ref_id_hash(ref_id); - - auto s = ref_id_hash.to_hex(); - std::string path = root_path_ + "/files/"; - for (td::uint32 i = 0; i < depth_; i++) { - path = path + s[2 * i] + s[2 * i + 1] + "/"; - if (create_dirs) { - td::mkdir(path).ensure(); - } - } - return path + s; -} - -void FileDb::store_file(RefId ref_id, td::BufferSlice data, td::Promise promise) { - auto ref_id_hash = get_ref_id_hash(ref_id); - auto R = get_block(ref_id_hash); - if (R.is_ok()) { - auto val = R.move_as_ok(); - promise.set_result(val.file_hash); - return; - } - - auto file_hash = sha256_bits256(data.as_slice()); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), file_hash, ref_id = std::move(ref_id), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &FileDb::store_file_continue, std::move(ref_id), file_hash, R.move_as_ok(), - std::move(promise)); - } - }); - td::actor::create_actor("writefile", root_path_ + "/tmp/", "", std::move(data), std::move(P)) - .release(); -} - -void FileDb::store_file_continue(RefId ref_id, FileHash file_hash, std::string res_path, - td::Promise promise) { - auto ref_id_hash = get_ref_id_hash(ref_id); - auto R = get_block(ref_id_hash); - if (R.is_ok()) { - td::unlink(res_path).ignore(); - auto val = R.move_as_ok(); - promise.set_result(val.file_hash); - return; - } - - auto path = get_file_name(ref_id, true); - td::rename(res_path, path).ensure(); - - auto empty = get_empty_ref_id_hash(); - auto ER = get_block(empty); - ER.ensure(); - auto E = ER.move_as_ok(); - - auto PR = get_block(E.prev); - PR.ensure(); - auto P = PR.move_as_ok(); - CHECK(P.next == empty); - - DbEntry D; - D.key = std::move(ref_id); - D.prev = E.prev; - D.next = empty; - D.file_hash = file_hash; - - E.prev = ref_id_hash; - P.next = ref_id_hash; - - if (P.is_empty()) { - E.next = ref_id_hash; - P.prev = ref_id_hash; - } - - kv_->begin_transaction().ensure(); - set_block(empty, std::move(E)); - set_block(D.prev, std::move(P)); - set_block(ref_id_hash, std::move(D)); - kv_->commit_transaction().ensure(); - - promise.set_value(std::move(file_hash)); -} - -void FileDb::load_file(RefId ref_id, td::Promise promise) { - auto ref_id_hash = get_ref_id_hash(ref_id); - auto R = get_block(ref_id_hash); - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - - auto v = R.move_as_ok(); - - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise), file_hash = v.file_hash](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - auto data = R.move_as_ok(); - if (file_hash != sha256_bits256(data.as_slice())) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "db error: bad file hash")); - } else { - promise.set_value(std::move(data)); - } - } - }); - - td::actor::create_actor("readfile", get_file_name(ref_id, false), 0, -1, 0, std::move(P)).release(); -} - -void FileDb::load_file_slice(RefId ref_id, td::int64 offset, td::int64 max_size, td::Promise promise) { - auto ref_id_hash = get_ref_id_hash(ref_id); - auto R = get_block(ref_id_hash); - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - - auto v = R.move_as_ok(); - - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_value(R.move_as_ok()); - } - }); - - td::actor::create_actor("readfile", get_file_name(ref_id, false), offset, max_size, 0, std::move(P)) - .release(); -} - -void FileDb::check_file(RefId ref_id, td::Promise promise) { - auto ref_id_hash = get_ref_id_hash(ref_id); - auto R = get_block(ref_id_hash); - if (R.is_error()) { - promise.set_result(false); - return; - } - promise.set_result(true); -} - -td::Slice FileDb::get_key(const RefIdHash& ref) { - return ref.as_slice(); -} - -td::Result FileDb::get_block(const RefIdHash& ref) { - std::string value; - auto R = kv_->get(get_key(ref), value); - R.ensure(); - if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) { - return td::Status::Error(ErrorCode::notready, "not in db"); - } - auto val = fetch_tl_object(td::BufferSlice{value}, true); - val.ensure(); - return DbEntry{val.move_as_ok()}; -} - -void FileDb::set_block(const RefIdHash& ref, DbEntry entry) { - DCHECK(ref == get_ref_id_hash(entry.key)); - kv_->set(get_key(ref), entry.release()).ensure(); -} - -FileDb::FileDb(td::actor::ActorId root_db, std::string root_path, td::uint32 depth, bool is_archive) - : root_db_(root_db), root_path_(root_path), depth_(depth), is_archive_(is_archive) { -} - -void FileDb::start_up() { - td::mkdir(root_path_).ensure(); - db_path_ = root_path_ + "/db/"; - - kv_ = std::make_shared(td::RocksDb::open(db_path_).move_as_ok()); - td::mkdir(root_path_ + "/files/").ensure(); - td::mkdir(root_path_ + "/tmp/").ensure(); - - last_gc_ = get_empty_ref_id_hash(); - alarm_timestamp() = td::Timestamp::in(0.01); - - auto R = get_block(last_gc_); - if (R.is_error()) { - DbEntry e{get_empty_ref_id(), last_gc_, last_gc_, FileHash::zero()}; - kv_->begin_transaction().ensure(); - set_block(last_gc_, std::move(e)); - kv_->commit_transaction().ensure(); - } -} - -void FileDb::alarm() { - auto R = get_block(last_gc_); - R.ensure(); - auto N = R.move_as_ok(); - if (N.is_empty()) { - last_gc_ = N.next; - alarm_timestamp() = td::Timestamp::in(0.01); - return; - } - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { - if (R.is_error()) { - td::actor::send_closure(SelfId, &FileDb::skip_gc); - } else { - auto value = R.move_as_ok(); - if (!value) { - td::actor::send_closure(SelfId, &FileDb::skip_gc); - } else { - td::actor::send_closure(SelfId, &FileDb::gc); - } - } - }); - td::actor::send_closure(root_db_, &RootDb::allow_gc, std::move(N.key), is_archive_, std::move(P)); -} - -void FileDb::gc() { - auto FR = get_block(last_gc_); - FR.ensure(); - auto F = FR.move_as_ok(); - - auto PR = get_block(F.prev); - PR.ensure(); - auto P = PR.move_as_ok(); - auto NR = get_block(F.next); - NR.ensure(); - auto N = NR.move_as_ok(); - - P.next = F.next; - N.prev = F.prev; - if (P.is_empty() && N.is_empty()) { - P.prev = P.next; - N.next = N.prev; - } - - auto name = get_file_name(F.key, false); - auto S = td::unlink(name); - if (S.is_error()) { - LOG(WARNING) << "failed to delete " << name; - } - - kv_->begin_transaction().ensure(); - kv_->erase(last_gc_.as_slice()).ensure(); - set_block(F.prev, std::move(P)); - set_block(F.next, std::move(N)); - kv_->commit_transaction().ensure(); - alarm_timestamp() = td::Timestamp::now(); - - DCHECK(get_block(last_gc_).is_error()); - last_gc_ = F.next; -} - -void FileDb::skip_gc() { - auto FR = get_block(last_gc_); - FR.ensure(); - auto F = FR.move_as_ok(); - last_gc_ = F.next; - alarm_timestamp() = td::Timestamp::in(0.01); -} - -td::BufferSlice FileDb::DbEntry::release() { - return create_serialize_tl_object(get_ref_id_tl(key), prev, next, file_hash); -} - -bool FileDb::DbEntry::is_empty() const { - return (key.get_offset() == key.offset()); -} - -FileDb::RefIdHash FileDb::get_ref_id_hash(const RefId& ref) { - FileHash x; - ref.visit([&](const auto& obj) { x = obj.hash(); }); - return x; -} - -tl_object_ptr FileDb::get_ref_id_tl(const RefId& ref) { - tl_object_ptr x; - ref.visit([&](const auto& obj) { x = obj.tl(); }); - return x; -} - -FileDb::RefId FileDb::get_ref_from_tl(const ton_api::db_filedb_Key& from) { - RefId ref_id{fileref::Empty{}}; - ton_api::downcast_call( - const_cast(from), - td::overloaded( - [&](const ton_api::db_filedb_key_empty& key) { ref_id = fileref::Empty{}; }, - [&](const ton_api::db_filedb_key_blockFile& key) { ref_id = fileref::Block{create_block_id(key.block_id_)}; }, - [&](const ton_api::db_filedb_key_zeroStateFile& key) { - ref_id = fileref::ZeroState{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_persistentStateFile& key) { - ref_id = - fileref::PersistentState{create_block_id(key.block_id_), create_block_id(key.masterchain_block_id_)}; - }, - [&](const ton_api::db_filedb_key_proof& key) { ref_id = fileref::Proof{create_block_id(key.block_id_)}; }, - [&](const ton_api::db_filedb_key_proofLink& key) { - ref_id = fileref::ProofLink{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_signatures& key) { - ref_id = fileref::Signatures{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_candidate& key) { - ref_id = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_), - key.id_->collated_data_file_hash_}; - }, - [&](const ton_api::db_filedb_key_blockInfo& key) { - ref_id = fileref::BlockInfo{create_block_id(key.block_id_)}; - })); - return ref_id; -} - -FileDb::RefId FileDb::get_empty_ref_id() { - return fileref::Empty(); -} - -FileDb::RefIdHash FileDb::get_empty_ref_id_hash() { - if (empty_.is_zero()) { - empty_ = get_ref_id_hash(get_empty_ref_id()); - } - return empty_; -} - -FileDb::DbEntry::DbEntry(tl_object_ptr entry) - : key(FileDb::get_ref_from_tl(*entry->key_.get())) - , prev(entry->prev_) - , next(entry->next_) - , file_hash(entry->file_hash_) { -} - -void FileDb::prepare_stats(td::Promise>> promise) { - std::vector> rocksdb_stats; - auto stats = kv_->stats(); - if (stats.size() == 0) { - promise.set_value(std::move(rocksdb_stats)); - return; - } - size_t pos = 0; - while (pos < stats.size()) { - while (pos < stats.size() && - (stats[pos] == ' ' || stats[pos] == '\n' || stats[pos] == '\r' || stats[pos] == '\t')) { - pos++; - } - auto p = pos; - if (pos == stats.size()) { - break; - } - while (stats[pos] != '\n' && stats[pos] != '\r' && stats[pos] != ' ' && stats[pos] != '\t' && pos < stats.size()) { - pos++; - } - auto name = stats.substr(p, pos - p); - if (stats[pos] == '\n' || pos == stats.size()) { - rocksdb_stats.emplace_back(name, ""); - continue; - } - while (pos < stats.size() && - (stats[pos] == ' ' || stats[pos] == '\n' || stats[pos] == '\r' || stats[pos] == '\t')) { - pos++; - } - p = pos; - while (stats[pos] != '\n' && stats[pos] != '\r' && pos < stats.size()) { - pos++; - } - auto value = stats.substr(p, pos - p); - rocksdb_stats.emplace_back(name, value); - } - promise.set_value(std::move(rocksdb_stats)); -} - -} // namespace validator - -} // namespace ton diff --git a/validator/db/filedb.hpp b/validator/db/filedb.hpp deleted file mode 100644 index 3adc4347..00000000 --- a/validator/db/filedb.hpp +++ /dev/null @@ -1,218 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#pragma once - -#include "ton/ton-types.h" -#include "td/actor/actor.h" -#include "validator/interfaces/shard.h" -#include "td/db/KeyValue.h" -#include "ton/ton-tl.hpp" - -namespace ton { - -namespace validator { - -namespace fileref { - -class Empty { - public: - tl_object_ptr tl() const { - return create_tl_object(); - } - FileHash hash() const { - return create_hash_tl_object(); - } -}; - -class Block { - public: - tl_object_ptr tl() const { - return create_tl_object(create_tl_block_id(block_id)); - } - FileHash hash() const { - return create_hash_tl_object(create_tl_block_id(block_id)); - } - - BlockIdExt block_id; -}; - -class ZeroState { - public: - tl_object_ptr tl() const { - return create_tl_object(create_tl_block_id(block_id)); - } - FileHash hash() const { - return create_hash_tl_object(create_tl_block_id(block_id)); - } - - BlockIdExt block_id; -}; - -class PersistentState { - public: - tl_object_ptr tl() const { - return create_tl_object(create_tl_block_id(block_id), - create_tl_block_id(masterchain_block_id)); - } - FileHash hash() const { - return create_hash_tl_object(create_tl_block_id(block_id), - create_tl_block_id(masterchain_block_id)); - } - - BlockIdExt block_id; - BlockIdExt masterchain_block_id; -}; - -class Proof { - public: - tl_object_ptr tl() const { - return create_tl_object(create_tl_block_id(block_id)); - } - FileHash hash() const { - return create_hash_tl_object(create_tl_block_id(block_id)); - } - - BlockIdExt block_id; -}; - -class ProofLink { - public: - tl_object_ptr tl() const { - return create_tl_object(create_tl_block_id(block_id)); - } - FileHash hash() const { - return create_hash_tl_object(create_tl_block_id(block_id)); - } - - BlockIdExt block_id; -}; - -class Signatures { - public: - tl_object_ptr tl() const { - return create_tl_object(create_tl_block_id(block_id)); - } - FileHash hash() const { - return create_hash_tl_object(create_tl_block_id(block_id)); - } - - BlockIdExt block_id; -}; - -class Candidate { - public: - tl_object_ptr tl() const { - return create_tl_object( - create_tl_object(source.tl(), create_tl_block_id(block_id), collated_data_file_hash)); - } - FileHash hash() const { - return create_hash_tl_object( - create_tl_object(source.tl(), create_tl_block_id(block_id), collated_data_file_hash)); - } - - PublicKey source; - BlockIdExt block_id; - FileHash collated_data_file_hash; -}; - -class BlockInfo { - public: - tl_object_ptr tl() const { - return create_tl_object(create_tl_block_id(block_id)); - } - FileHash hash() const { - return create_hash_tl_object(create_tl_block_id(block_id)); - } - - BlockIdExt block_id; -}; -}; // namespace fileref - -class RootDb; - -class FileDb : public td::actor::Actor { - public: - using RefId = - td::Variant; - using RefIdHash = td::Bits256; - - void store_file(RefId ref_id, td::BufferSlice data, td::Promise promise); - void store_file_continue(RefId ref_id, FileHash file_hash, std::string path, td::Promise promise); - void load_file(RefId ref_id, td::Promise promise); - void load_file_slice(RefId ref_id, td::int64 offset, td::int64 max_size, td::Promise promise); - void check_file(RefId ref_id, td::Promise promise); - - void prepare_stats(td::Promise>> promise); - - void start_up() override; - void alarm() override; - - void gc(); - void skip_gc(); - - FileDb(td::actor::ActorId root_db, std::string root_path, td::uint32 depth, bool is_archive); - - private: - struct DbEntry { - RefId key; - RefIdHash prev; - RefIdHash next; - FileHash file_hash; - - DbEntry(tl_object_ptr entry); - DbEntry() { - } - DbEntry(RefId key, RefIdHash prev, RefIdHash next, FileHash file_hash) - : key(std::move(key)), prev(prev), next(next), file_hash(file_hash) { - } - td::BufferSlice release(); - bool is_empty() const; - }; - - static RefIdHash get_ref_id_hash(const RefId& ref); - static tl_object_ptr get_ref_id_tl(const RefId& ref); - static RefId get_ref_from_tl(const ton_api::db_filedb_Key& from); - static RefId get_empty_ref_id(); - RefIdHash get_empty_ref_id_hash(); - - std::string get_file_name(const RefId& ref, bool create_dirs); - td::Slice get_key(const RefIdHash& ref); - - td::Result get_block(const RefIdHash& ref_id); - void set_block(const RefIdHash& ref_id_hash, DbEntry entry); - - td::actor::ActorId root_db_; - - std::string root_path_; - std::string db_path_; - td::uint32 depth_; - - bool is_archive_; - - std::shared_ptr kv_; - - RefIdHash last_gc_; - RefIdHash empty_ = RefIdHash::zero(); -}; - -} // namespace validator - -} // namespace ton - diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp new file mode 100644 index 00000000..968ec552 --- /dev/null +++ b/validator/db/fileref.cpp @@ -0,0 +1,471 @@ +#include "fileref.hpp" +#include "auto/tl/ton_api.hpp" +#include "td/utils/overloaded.h" +#include "td/utils/misc.h" + +namespace ton { + +namespace validator { + +namespace { + +td::Result get_block_id(std::stringstream& ss) { + std::string token; + BlockId block_id; + std::getline(ss, token, '_'); + TRY_RESULT(w, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(shard, td::hex_to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(s, td::to_integer_safe(token)); + return BlockId{w, shard, s}; +} + +td::Result get_token_hash(std::stringstream& ss) { + std::string token; + std::getline(ss, token, '_'); + if (token.size() != 64) { + return td::Status::Error(ErrorCode::protoviolation, "hash must have exactly 64 hexdigits"); + } + + TRY_RESULT(v, td::hex_decode(token)); + + FileHash r; + r.as_slice().copy_from(v); + return r; +} + +} // namespace + +namespace fileref { + +std::string Empty::filename() const { + return "empty"; +} + +Empty Empty::shortref() const { + return *this; +} + +std::string Empty::filename_short() const { + return "empty"; +} + +BlockShort Block::shortref() const { + return BlockShort{block_id.id, hash()}; +} + +std::string Block::filename() const { + return PSTRING() << "block_" << block_id.to_str(); +} + +std::string Block::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.id.shard)); + return PSTRING() << "block_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" + << hash().to_hex(); +} + +std::string BlockShort::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.shard)); + return PSTRING() << "block_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); +} + +ZeroStateShort ZeroState::shortref() const { + return ZeroStateShort{block_id.id.workchain, hash()}; +} + +std::string ZeroState::filename() const { + return PSTRING() << "zerostate_" << block_id.to_str(); +} + +std::string ZeroState::filename_short() const { + return PSTRING() << "zerostate_" << block_id.id.workchain << "_" << hash().to_hex(); +} + +std::string ZeroStateShort::filename_short() const { + return PSTRING() << "zerostate_" << workchain << "_" << hash().to_hex(); +} + +PersistentStateShort PersistentState::shortref() const { + return PersistentStateShort{block_id.shard_full(), masterchain_block_id.seqno(), hash()}; +} + +std::string PersistentState::filename() const { + return PSTRING() << "state_" << masterchain_block_id.to_str() << "_" << block_id.to_str(); +} + +std::string PersistentState::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.id.shard)); + return PSTRING() << "state_" << masterchain_block_id.seqno() << "_" << block_id.id.workchain << "_" << s << "_" + << hash().to_hex(); +} + +std::string PersistentStateShort::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(shard_id.shard)); + return PSTRING() << "state_" << masterchain_seqno << "_" << shard_id.workchain << "_" << s << "_" << hash().to_hex(); +} + +ProofShort Proof::shortref() const { + return ProofShort{block_id.id, hash()}; +} + +std::string Proof::filename() const { + return PSTRING() << "proof_" << block_id.to_str(); +} + +std::string Proof::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.id.shard)); + return PSTRING() << "proof_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" + << hash().to_hex(); +} + +std::string ProofShort::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.shard)); + return PSTRING() << "proof_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); +} + +ProofLinkShort ProofLink::shortref() const { + return ProofLinkShort{block_id.id, hash()}; +} + +std::string ProofLink::filename() const { + return PSTRING() << "prooflink_" << block_id.to_str(); +} + +std::string ProofLink::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.id.shard)); + return PSTRING() << "prooflink_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" + << hash().to_hex(); +} + +std::string ProofLinkShort::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.shard)); + return PSTRING() << "prooflink_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); +} + +SignaturesShort Signatures::shortref() const { + return SignaturesShort{block_id.id, hash()}; +} + +std::string Signatures::filename() const { + return PSTRING() << "signatures_" << block_id.to_str(); +} + +std::string Signatures::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.id.shard)); + return PSTRING() << "signatures_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" + << hash().to_hex(); +} + +std::string SignaturesShort::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.shard)); + return PSTRING() << "signatures_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" + << hash().to_hex(); +} + +CandidateShort Candidate::shortref() const { + return CandidateShort{block_id.id, hash()}; +} + +std::string Candidate::filename() const { + return PSTRING() << "candidate_" << block_id.to_str() << "_" << collated_data_file_hash.to_hex() << "_" + << td::base64url_encode(source.export_as_slice()); +} + +std::string Candidate::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.id.shard)); + return PSTRING() << "candidate_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" + << hash().to_hex(); +} + +std::string CandidateShort::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.shard)); + return PSTRING() << "candidate_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); +} + +BlockInfoShort BlockInfo::shortref() const { + return BlockInfoShort{block_id.id, hash()}; +} + +std::string BlockInfo::filename() const { + return PSTRING() << "info_" << block_id.to_str(); +} + +std::string BlockInfo::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.id.shard)); + return PSTRING() << "info_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" + << hash().to_hex(); +} + +std::string BlockInfoShort::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.shard)); + return PSTRING() << "info_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); +} + +} // namespace fileref + +FileReference::FileReference(tl_object_ptr key) { + ton_api::downcast_call( + *key.get(), + td::overloaded( + [&](const ton_api::db_filedb_key_empty& key) { ref_ = fileref::Empty{}; }, + [&](const ton_api::db_filedb_key_blockFile& key) { ref_ = fileref::Block{create_block_id(key.block_id_)}; }, + [&](const ton_api::db_filedb_key_zeroStateFile& key) { + ref_ = fileref::ZeroState{create_block_id(key.block_id_)}; + }, + [&](const ton_api::db_filedb_key_persistentStateFile& key) { + ref_ = fileref::PersistentState{create_block_id(key.block_id_), create_block_id(key.masterchain_block_id_)}; + }, + [&](const ton_api::db_filedb_key_proof& key) { ref_ = fileref::Proof{create_block_id(key.block_id_)}; }, + [&](const ton_api::db_filedb_key_proofLink& key) { + ref_ = fileref::ProofLink{create_block_id(key.block_id_)}; + }, + [&](const ton_api::db_filedb_key_signatures& key) { + ref_ = fileref::Signatures{create_block_id(key.block_id_)}; + }, + [&](const ton_api::db_filedb_key_candidate& key) { + ref_ = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_), + key.id_->collated_data_file_hash_}; + }, + [&](const ton_api::db_filedb_key_blockInfo& key) { + ref_ = fileref::BlockInfo{create_block_id(key.block_id_)}; + })); +} + +FileReferenceShort FileReference::shortref() const { + FileReferenceShort h; + ref_.visit([&](const auto& obj) { h = obj.shortref(); }); + return h; +} + +td::Bits256 FileReference::hash() const { + FileHash h; + ref_.visit([&](const auto& obj) { h = obj.hash(); }); + return h; +} + +td::Bits256 FileReferenceShort::hash() const { + FileHash h; + ref_.visit([&](const auto& obj) { h = obj.hash(); }); + return h; +} + +ShardIdFull FileReference::shard() const { + ShardIdFull h; + ref_.visit([&](const auto& obj) { h = obj.shard(); }); + return h; +} + +ShardIdFull FileReferenceShort::shard() const { + ShardIdFull h; + ref_.visit([&](const auto& obj) { h = obj.shard(); }); + return h; +} + +std::string FileReference::filename() const { + std::string h; + ref_.visit([&](const auto& obj) { h = obj.filename(); }); + return h; +} + +std::string FileReference::filename_short() const { + std::string h; + ref_.visit([&](const auto& obj) { h = obj.filename_short(); }); + return h; +} + +std::string FileReferenceShort::filename_short() const { + std::string h; + ref_.visit([&](const auto& obj) { h = obj.filename_short(); }); + return h; +} + +td::Result FileReference::create(std::string filename) { + std::stringstream ss{filename}; + + std::string token; + std::getline(ss, token, '_'); + + if (token == "empty") { + if (ss.eof()) { + return fileref::Empty{}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "block") { + std::getline(ss, token, '_'); + TRY_RESULT(block_id, BlockIdExt::from_str(token)); + if (ss.eof()) { + return fileref::Block{block_id}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "zerostate") { + std::getline(ss, token, '_'); + TRY_RESULT(block_id, BlockIdExt::from_str(token)); + if (ss.eof()) { + return fileref::ZeroState{block_id}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "state") { + std::getline(ss, token, '_'); + TRY_RESULT(masterchain_block_id, BlockIdExt::from_str(token)); + std::getline(ss, token, '_'); + TRY_RESULT(block_id, BlockIdExt::from_str(token)); + if (ss.eof()) { + return fileref::PersistentState{block_id, masterchain_block_id}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "proof") { + std::getline(ss, token, '_'); + TRY_RESULT(block_id, BlockIdExt::from_str(token)); + if (ss.eof()) { + return fileref::Proof{block_id}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "prooflink") { + std::getline(ss, token, '_'); + TRY_RESULT(block_id, BlockIdExt::from_str(token)); + if (ss.eof()) { + return fileref::ProofLink{block_id}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "signatures") { + std::getline(ss, token, '_'); + TRY_RESULT(block_id, BlockIdExt::from_str(token)); + if (ss.eof()) { + return fileref::Signatures{block_id}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "candidate") { + std::getline(ss, token, '_'); + TRY_RESULT(block_id, BlockIdExt::from_str(token)); + TRY_RESULT(col_hash, get_token_hash(ss)); + + std::string rem = ss.str(); + + TRY_RESULT(source_s, td::base64url_decode(rem)); + TRY_RESULT(source, PublicKey::import(source_s)); + return fileref::Candidate{source, block_id, col_hash}; + } else if (token == "info") { + std::getline(ss, token, '_'); + TRY_RESULT(block_id, BlockIdExt::from_str(token)); + if (ss.eof()) { + return fileref::BlockInfo{block_id}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else { + return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "unknown prefix '" << token << "'"); + } +} + +td::Result FileReferenceShort::create(std::string filename) { + std::stringstream ss{filename}; + + std::string token; + std::getline(ss, token, '_'); + + if (token == "empty") { + if (ss.eof()) { + return fileref::Empty{}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "block") { + TRY_RESULT(block_id, get_block_id(ss)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (ss.eof()) { + return fileref::BlockShort{block_id, vhash}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "zerostate") { + std::getline(ss, token, '_'); + TRY_RESULT(workchain, td::to_integer_safe(token)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (ss.eof()) { + return fileref::ZeroStateShort{workchain, vhash}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "state") { + std::getline(ss, token, '_'); + TRY_RESULT(masterchain_seqno, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(workchain, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(shard, td::hex_to_integer_safe(token)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (ss.eof()) { + return fileref::PersistentStateShort{ShardIdFull{workchain, shard}, masterchain_seqno, vhash}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "proof") { + TRY_RESULT(block_id, get_block_id(ss)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (ss.eof()) { + return fileref::ProofShort{block_id, vhash}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "prooflink") { + TRY_RESULT(block_id, get_block_id(ss)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (ss.eof()) { + return fileref::ProofLinkShort{block_id, vhash}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "signatures") { + TRY_RESULT(block_id, get_block_id(ss)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (ss.eof()) { + return fileref::SignaturesShort{block_id, vhash}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "candidate") { + TRY_RESULT(block_id, get_block_id(ss)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (ss.eof()) { + return fileref::CandidateShort{block_id, vhash}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else if (token == "info") { + TRY_RESULT(block_id, get_block_id(ss)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (ss.eof()) { + return fileref::BlockInfoShort{block_id, vhash}; + } else { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + } else { + return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "unknown prefix '" << token << "'"); + } +} + +} // namespace validator + +} // namespace ton diff --git a/validator/db/fileref.hpp b/validator/db/fileref.hpp new file mode 100644 index 00000000..9b012dc8 --- /dev/null +++ b/validator/db/fileref.hpp @@ -0,0 +1,372 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#pragma once + +#include "ton/ton-types.h" +#include "td/actor/actor.h" +#include "validator/interfaces/shard.h" +#include "td/db/KeyValue.h" +#include "ton/ton-tl.hpp" + +namespace ton { + +namespace validator { + +namespace fileref { + +class Empty { + public: + tl_object_ptr tl() const { + return create_tl_object(); + } + FileHash hash() const { + return create_hash_tl_object(); + } + ShardIdFull shard() const { + return ShardIdFull{masterchainId}; + } + std::string filename() const; + std::string filename_short() const; + Empty shortref() const; +}; + +class BlockShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + std::string filename_short() const; + + BlockId block_id; + FileHash hashv; +}; + +class Block { + public: + tl_object_ptr tl() const { + return create_tl_object(create_tl_block_id(block_id)); + } + FileHash hash() const { + return create_hash_tl_object(create_tl_block_id(block_id)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + BlockShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + BlockIdExt block_id; +}; + +class ZeroStateShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return ShardIdFull{workchain, shardIdAll}; + } + std::string filename_short() const; + + WorkchainId workchain; + FileHash hashv; +}; + +class ZeroState { + public: + tl_object_ptr tl() const { + return create_tl_object(create_tl_block_id(block_id)); + } + FileHash hash() const { + return create_hash_tl_object(create_tl_block_id(block_id)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + ZeroStateShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + BlockIdExt block_id; +}; + +class PersistentStateShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return shard_id; + } + std::string filename_short() const; + + ShardIdFull shard_id; + BlockSeqno masterchain_seqno; + FileHash hashv; +}; + +class PersistentState { + public: + tl_object_ptr tl() const { + return create_tl_object(create_tl_block_id(block_id), + create_tl_block_id(masterchain_block_id)); + } + FileHash hash() const { + return create_hash_tl_object(create_tl_block_id(block_id), + create_tl_block_id(masterchain_block_id)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + PersistentStateShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + BlockIdExt block_id; + BlockIdExt masterchain_block_id; +}; + +class ProofShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + std::string filename_short() const; + + BlockId block_id; + FileHash hashv; +}; + +class Proof { + public: + tl_object_ptr tl() const { + return create_tl_object(create_tl_block_id(block_id)); + } + FileHash hash() const { + return create_hash_tl_object(create_tl_block_id(block_id)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + ProofShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + BlockIdExt block_id; +}; + +class ProofLinkShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + std::string filename_short() const; + + BlockId block_id; + FileHash hashv; +}; + +class ProofLink { + public: + tl_object_ptr tl() const { + return create_tl_object(create_tl_block_id(block_id)); + } + FileHash hash() const { + return create_hash_tl_object(create_tl_block_id(block_id)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + ProofLinkShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + BlockIdExt block_id; +}; + +class SignaturesShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + std::string filename_short() const; + + BlockId block_id; + FileHash hashv; +}; + +class Signatures { + public: + tl_object_ptr tl() const { + return create_tl_object(create_tl_block_id(block_id)); + } + FileHash hash() const { + return create_hash_tl_object(create_tl_block_id(block_id)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + SignaturesShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + BlockIdExt block_id; +}; + +class CandidateShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + std::string filename_short() const; + + BlockId block_id; + FileHash hashv; +}; + +class Candidate { + public: + tl_object_ptr tl() const { + return create_tl_object( + create_tl_object(source.tl(), create_tl_block_id(block_id), collated_data_file_hash)); + } + FileHash hash() const { + return create_hash_tl_object( + create_tl_object(source.tl(), create_tl_block_id(block_id), collated_data_file_hash)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + CandidateShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + PublicKey source; + BlockIdExt block_id; + FileHash collated_data_file_hash; +}; + +class BlockInfoShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + std::string filename_short() const; + + BlockId block_id; + FileHash hashv; +}; + +class BlockInfo { + public: + tl_object_ptr tl() const { + return create_tl_object(create_tl_block_id(block_id)); + } + FileHash hash() const { + return create_hash_tl_object(create_tl_block_id(block_id)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + BlockInfoShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + BlockIdExt block_id; +}; + +}; // namespace fileref + +class FileReferenceShort { + private: + td::Variant + ref_; + + public: + template + FileReferenceShort(T x) : ref_(std::move(x)) { + } + FileReferenceShort() : ref_(fileref::Empty{}) { + } + + static td::Result create(std::string filename); + + auto &ref() { + return ref_; + } + + td::Bits256 hash() const; + ShardIdFull shard() const; + std::string filename_short() const; +}; + +class FileReference { + private: + td::Variant + ref_; + + public: + template + FileReference(T x) : ref_(std::move(x)) { + } + FileReference() : ref_(fileref::Empty{}) { + } + FileReference(tl_object_ptr key); + + static td::Result create(std::string filename); + + auto &ref() { + return ref_; + } + + FileReferenceShort shortref() const; + + tl_object_ptr tl() const; + td::Bits256 hash() const; + ShardIdFull shard() const; + std::string filename() const; + std::string filename_short() const; +}; + +} // namespace validator + +} // namespace ton + diff --git a/validator/db/ltdb.cpp b/validator/db/ltdb.cpp deleted file mode 100644 index eea2aa63..00000000 --- a/validator/db/ltdb.cpp +++ /dev/null @@ -1,310 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#include "ltdb.hpp" -#include "rootdb.hpp" -#include "auto/tl/ton_api.h" -#include "ton/ton-tl.hpp" -#include "ton/ton-io.hpp" -#include "td/db/RocksDb.h" - -namespace ton { - -namespace validator { - -void LtDb::add_new_block(BlockIdExt block_id, LogicalTime lt, UnixTime ts, td::Promise promise) { - auto key = get_desc_key(block_id.shard_full()); - - std::string value; - auto R = kv_->get(key.as_slice(), value); - R.ensure(); - tl_object_ptr v; - bool add_shard = false; - if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) { - auto F = fetch_tl_object(td::BufferSlice{value}, true); - F.ensure(); - v = F.move_as_ok(); - } else { - v = create_tl_object(1, 1, 0, 0, 0); - add_shard = true; - } - if (block_id.id.seqno <= static_cast(v->last_seqno_) || lt <= static_cast(v->last_lt_) || - ts <= static_cast(v->last_ts_)) { - promise.set_value(td::Unit()); - return; - } - auto db_value = create_serialize_tl_object(create_tl_block_id(block_id), lt, ts); - auto db_key = get_el_key(block_id.shard_full(), v->last_idx_++); - auto status_key = create_serialize_tl_object(); - v->last_seqno_ = block_id.id.seqno; - v->last_lt_ = lt; - v->last_ts_ = ts; - - td::uint32 idx = 0; - if (add_shard) { - auto G = kv_->get(status_key.as_slice(), value); - G.ensure(); - if (G.move_as_ok() == td::KeyValue::GetStatus::NotFound) { - idx = 0; - } else { - auto F = fetch_tl_object(value, true); - F.ensure(); - auto f = F.move_as_ok(); - idx = f->total_shards_; - } - } - - kv_->begin_transaction().ensure(); - kv_->set(key, serialize_tl_object(v, true)).ensure(); - kv_->set(db_key, db_value.as_slice()).ensure(); - if (add_shard) { - auto shard_key = create_serialize_tl_object(idx); - auto shard_value = create_serialize_tl_object(block_id.id.workchain, block_id.id.shard); - kv_->set(status_key.as_slice(), create_serialize_tl_object(idx + 1)).ensure(); - kv_->set(shard_key.as_slice(), shard_value.as_slice()).ensure(); - } - kv_->commit_transaction().ensure(); - - promise.set_value(td::Unit()); -} - -void LtDb::get_block_common(AccountIdPrefixFull account_id, - std::function compare_desc, - std::function compare, bool exact, - td::Promise promise) { - bool f = false; - BlockIdExt block_id; - td::uint32 ls = 0; - for (td::uint32 len = 0; len <= 60; len++) { - auto s = shard_prefix(account_id, len); - auto key = get_desc_key(s); - std::string value; - auto F = kv_->get(key, value); - F.ensure(); - if (F.move_as_ok() == td::KeyValue::GetStatus::NotFound) { - if (!f) { - continue; - } else { - break; - } - } - f = true; - auto G = fetch_tl_object(td::BufferSlice{value}, true); - G.ensure(); - auto g = G.move_as_ok(); - if (compare_desc(*g.get()) > 0) { - continue; - } - td::uint32 l = g->first_idx_ - 1; - BlockIdExt lseq; - td::uint32 r = g->last_idx_; - BlockIdExt rseq; - while (r - l > 1) { - auto x = (r + l) / 2; - auto db_key = get_el_key(s, x); - F = kv_->get(db_key, value); - F.ensure(); - CHECK(F.move_as_ok() == td::KeyValue::GetStatus::Ok); - auto E = fetch_tl_object(td::BufferSlice{value}, true); - E.ensure(); - auto e = E.move_as_ok(); - int cmp_val = compare(*e.get()); - - if (cmp_val < 0) { - rseq = create_block_id(e->id_); - r = x; - } else if (cmp_val > 0) { - lseq = create_block_id(e->id_); - l = x; - } else { - promise.set_value(create_block_id(e->id_)); - return; - } - } - if (rseq.is_valid()) { - if (!block_id.is_valid()) { - block_id = rseq; - } else if (block_id.id.seqno > rseq.id.seqno) { - block_id = rseq; - } - } - if (lseq.is_valid()) { - if (ls < lseq.id.seqno) { - ls = lseq.id.seqno; - } - } - if (block_id.is_valid() && ls + 1 == block_id.id.seqno) { - if (!exact) { - promise.set_value(std::move(block_id)); - } else { - promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found")); - } - return; - } - } - if (!exact && block_id.is_valid()) { - promise.set_value(std::move(block_id)); - } else { - promise.set_error(td::Status::Error(ErrorCode::notready, "ltdb: block not found")); - } -} - -void LtDb::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise) { - return get_block_common( - account_id, - [lt](ton_api::db_lt_desc_value &w) { - return lt > static_cast(w.last_lt_) ? 1 : lt == static_cast(w.last_lt_) ? 0 : -1; - }, - [lt](ton_api::db_lt_el_value &w) { - return lt > static_cast(w.lt_) ? 1 : lt == static_cast(w.lt_) ? 0 : -1; - }, - false, std::move(promise)); -} - -void LtDb::get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise promise) { - return get_block_common( - account_id, - [seqno](ton_api::db_lt_desc_value &w) { - return seqno > static_cast(w.last_seqno_) - ? 1 - : seqno == static_cast(w.last_seqno_) ? 0 : -1; - }, - [seqno](ton_api::db_lt_el_value &w) { - return seqno > static_cast(w.id_->seqno_) - ? 1 - : seqno == static_cast(w.id_->seqno_) ? 0 : -1; - }, - true, std::move(promise)); -} - -void LtDb::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise) { - return get_block_common( - account_id, - [ts](ton_api::db_lt_desc_value &w) { - return ts > static_cast(w.last_ts_) ? 1 : ts == static_cast(w.last_ts_) ? 0 : -1; - }, - [ts](ton_api::db_lt_el_value &w) { - return ts > static_cast(w.ts_) ? 1 : ts == static_cast(w.ts_) ? 0 : -1; - }, - false, std::move(promise)); -} - -td::BufferSlice LtDb::get_desc_key(ShardIdFull shard) { - return create_serialize_tl_object(shard.workchain, shard.shard); -} - -td::BufferSlice LtDb::get_el_key(ShardIdFull shard, td::uint32 idx) { - return create_serialize_tl_object(shard.workchain, shard.shard, idx); -} - -void LtDb::start_up() { - kv_ = std::make_shared(td::RocksDb::open(db_path_).move_as_ok()); -} - -void LtDb::truncate_workchain(ShardIdFull shard, td::Ref state) { - auto key = get_desc_key(shard); - std::string value; - auto R = kv_->get(key, value); - R.ensure(); - CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok); - auto F = fetch_tl_object(td::BufferSlice{value}, true); - F.ensure(); - auto f = F.move_as_ok(); - - auto shards = state->get_shards(); - BlockSeqno seqno = 0; - if (shard.is_masterchain()) { - seqno = state->get_seqno(); - } else { - for (auto s : shards) { - if (shard_intersects(s->shard(), shard)) { - seqno = s->top_block_id().seqno(); - break; - } - } - } - - while (f->last_idx_ > f->first_idx_) { - auto db_key = get_el_key(shard, f->last_idx_ - 1); - R = kv_->get(db_key, value); - R.ensure(); - CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok); - auto E = fetch_tl_object(td::BufferSlice{value}, true); - E.ensure(); - auto e = E.move_as_ok(); - - bool to_delete = static_cast(e->id_->seqno_) > seqno; - - if (!to_delete) { - break; - } else { - f->last_idx_--; - kv_->erase(db_key).ensure(); - } - } - - if (f->first_idx_ == f->last_idx_) { - f->last_ts_ = 0; - f->last_lt_ = 0; - f->last_seqno_ = 0; - } - - kv_->set(key, serialize_tl_object(f, true)).ensure(); -} - -void LtDb::truncate(td::Ref state, td::Promise promise) { - auto status_key = create_serialize_tl_object(); - td::Result R; - td::uint32 total_shards = 0; - { - std::string value; - R = kv_->get(status_key.as_slice(), value); - R.ensure(); - if (R.move_as_ok() == td::KeyValue::GetStatus::NotFound) { - promise.set_value(td::Unit()); - return; - } - auto F = fetch_tl_object(value, true); - F.ensure(); - auto f = F.move_as_ok(); - total_shards = f->total_shards_; - if (total_shards == 0) { - promise.set_value(td::Unit()); - return; - } - } - kv_->begin_transaction().ensure(); - for (td::uint32 idx = 0; idx < total_shards; idx++) { - auto shard_key = create_serialize_tl_object(idx); - std::string value; - R = kv_->get(shard_key.as_slice(), value); - R.ensure(); - CHECK(R.move_as_ok() == td::KeyValue::GetStatus::Ok); - auto F = fetch_tl_object(value, true); - F.ensure(); - auto f = F.move_as_ok(); - - truncate_workchain(ShardIdFull{f->workchain_, static_cast(f->shard_)}, state); - } - kv_->commit_transaction().ensure(); - promise.set_value(td::Unit()); -} - -} // namespace validator - -} // namespace ton diff --git a/validator/db/ltdb.hpp b/validator/db/ltdb.hpp deleted file mode 100644 index 4c7651ec..00000000 --- a/validator/db/ltdb.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#pragma once - -#include "td/actor/actor.h" -#include "td/db/KeyValueAsync.h" -#include "validator/interfaces/db.h" - -#include "ton/ton-types.h" - -#include "auto/tl/ton_api.h" - -namespace ton { - -namespace validator { - -class RootDb; - -class LtDb : public td::actor::Actor { - public: - void add_new_block(BlockIdExt block_id, LogicalTime lt, UnixTime ts, td::Promise promise); - void get_block_common(AccountIdPrefixFull account_id, - std::function compare_desc, - std::function compare, bool exact, - td::Promise promise); - void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise); - void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise); - void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise promise); - - void truncate_workchain(ShardIdFull shard, td::Ref state); - void truncate(td::Ref state, td::Promise promise); - - void start_up() override; - - LtDb(td::actor::ActorId root_db, std::string db_path) : root_db_(root_db), db_path_(std::move(db_path)) { - } - - private: - td::BufferSlice get_desc_key(ShardIdFull shard); - td::BufferSlice get_el_key(ShardIdFull shard, td::uint32 idx); - td::BufferSlice get_from_db(ShardIdFull shard, td::uint32 idx); - - std::shared_ptr kv_; - - td::actor::ActorId root_db_; - std::string db_path_; -}; - -} // namespace validator - -} // namespace ton diff --git a/validator/db/package.cpp b/validator/db/package.cpp index 76f504a3..e15767df 100644 --- a/validator/db/package.cpp +++ b/validator/db/package.cpp @@ -34,7 +34,7 @@ td::Status Package::truncate(td::uint64 size) { return fd_.truncate_to_current_position(size + header_size()); } -td::uint64 Package::append(std::string filename, td::Slice data) { +td::uint64 Package::append(std::string filename, td::Slice data, bool sync) { CHECK(data.size() <= max_data_size()); CHECK(filename.size() <= max_filename_size()); auto size = fd_.get_size().move_as_ok(); @@ -48,10 +48,16 @@ td::uint64 Package::append(std::string filename, td::Slice data) { size += filename.size(); CHECK(fd_.pwrite(data, size).move_as_ok() == data.size()); size += data.size(); - fd_.sync().ensure(); + if (sync) { + fd_.sync().ensure(); + } return orig_size - header_size(); } +void Package::sync() { + fd_.sync().ensure(); +} + td::uint64 Package::size() const { return fd_.get_size().move_as_ok() - header_size(); } @@ -140,4 +146,28 @@ td::Result Package::open(std::string path, bool read_only, bool create) return Package{std::move(fd)}; } +void Package::iterate(std::function func) { + td::uint64 p = 0; + + td::uint64 size = fd_.get_size().move_as_ok(); + if (size < header_size()) { + LOG(ERROR) << "too short archive"; + return; + } + size -= header_size(); + while (p != size) { + auto R = read(p); + if (R.is_error()) { + LOG(ERROR) << "broken archive: " << R.move_as_error(); + return; + } + auto q = R.move_as_ok(); + if (!func(q.first, q.second.clone(), p)) { + break; + } + + p = advance(p).move_as_ok(); + } +} + } // namespace ton diff --git a/validator/db/package.hpp b/validator/db/package.hpp index 50d6ad0a..b4236d05 100644 --- a/validator/db/package.hpp +++ b/validator/db/package.hpp @@ -14,24 +14,17 @@ class Package { td::Status truncate(td::uint64 size); - td::uint64 append(std::string filename, td::Slice data); + td::uint64 append(std::string filename, td::Slice data, bool sync = true); + void sync(); td::uint64 size() const; td::Result> read(td::uint64 offset) const; + td::Result advance(td::uint64 offset); + void iterate(std::function func); - struct Iterator { - td::uint64 offset; - Package &package; - - Iterator operator++(int); - const Iterator operator++(int) const; - td::Result> read() const; - }; - - Iterator begin(); - const Iterator begin() const; - Iterator end(); - const Iterator end() const; + td::FileFd &fd() { + return fd_; + } private: td::FileFd fd_; diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index a0e4d521..21cc196d 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -32,22 +32,22 @@ namespace ton { namespace validator { void RootDb::store_block_data(BlockHandle handle, td::Ref block, td::Promise promise) { - if (handle->moved_to_storage() || handle->moved_to_archive()) { + if (handle->received()) { promise.set_value(td::Unit()); return; } - auto id = block_db_.get(); - auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - handle->set_received(); - td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise)); - } - }); + auto P = td::PromiseCreator::lambda( + [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + handle->set_received(); + td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise)); + } + }); - td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::Block{handle->id()}}, block->data(), + td::actor::send_closure(archive_db_, &ArchiveManager::add_file, handle, fileref::Block{handle->id()}, block->data(), std::move(P)); } @@ -64,45 +64,34 @@ void RootDb::get_block_data(BlockHandle handle, td::Promise> } }); - if (handle->moved_to_archive()) { - td::actor::send_closure(new_archive_db_, &ArchiveManager::read, handle->unix_time(), handle->is_key_block(), - FileDb::RefId{fileref::Block{handle->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle->moved_to_storage() ? old_archive_db_.get() : file_db_.get(), &FileDb::load_file, - FileDb::RefId{fileref::Block{handle->id()}}, std::move(P)); - } + td::actor::send_closure(archive_db_, &ArchiveManager::get_file, handle, fileref::Block{handle->id()}, std::move(P)); } } void RootDb::store_block_signatures(BlockHandle handle, td::Ref data, td::Promise promise) { - if (handle->moved_to_storage() || handle->moved_to_archive()) { + if (handle->inited_signatures() || handle->moved_to_archive()) { promise.set_value(td::Unit()); return; } - auto id = block_db_.get(); - auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - handle->set_signatures(); - td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise)); - } - }); - - td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::Signatures{handle->id()}}, + auto P = td::PromiseCreator::lambda( + [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + handle->set_signatures(); + td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise)); + } + }); + td::actor::send_closure(archive_db_, &ArchiveManager::add_temp_file_short, fileref::Signatures{handle->id()}, data->serialize(), std::move(P)); } void RootDb::get_block_signatures(BlockHandle handle, td::Promise> promise) { - if (!handle->inited_signatures()) { + if (!handle->inited_signatures() || handle->moved_to_archive()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not in db")); } else { - if (handle->moved_to_storage() || handle->moved_to_archive()) { - promise.set_error(td::Status::Error(ErrorCode::error, "signatures already gc'd")); - return; - } auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); @@ -110,28 +99,28 @@ void RootDb::get_block_signatures(BlockHandle handle, td::Promiseid()}}, + td::actor::send_closure(archive_db_, &ArchiveManager::get_temp_file_short, fileref::Signatures{handle->id()}, std::move(P)); } } void RootDb::store_block_proof(BlockHandle handle, td::Ref proof, td::Promise promise) { - if (handle->moved_to_storage() || handle->moved_to_archive()) { + if (handle->inited_proof()) { promise.set_value(td::Unit()); return; } - auto id = block_db_.get(); - auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - handle->set_proof(); - td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise)); - } - }); + auto P = td::PromiseCreator::lambda( + [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + handle->set_proof(); + td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise)); + } + }); - td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::Proof{handle->id()}}, proof->data(), + td::actor::send_closure(archive_db_, &ArchiveManager::add_file, handle, fileref::Proof{handle->id()}, proof->data(), std::move(P)); } @@ -147,34 +136,27 @@ void RootDb::get_block_proof(BlockHandle handle, td::Promise> pro promise.set_result(create_proof(id, R.move_as_ok())); } }); - if (handle->moved_to_archive()) { - td::actor::send_closure(new_archive_db_, &ArchiveManager::read, handle->unix_time(), handle->is_key_block(), - FileDb::RefId{fileref::Proof{handle->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle->moved_to_storage() ? old_archive_db_.get() : file_db_.get(), &FileDb::load_file, - FileDb::RefId{fileref::Proof{handle->id()}}, std::move(P)); - } + td::actor::send_closure(archive_db_, &ArchiveManager::get_file, handle, fileref::Proof{handle->id()}, std::move(P)); } } void RootDb::store_block_proof_link(BlockHandle handle, td::Ref proof, td::Promise promise) { - if (handle->moved_to_storage() || handle->moved_to_archive()) { + if (handle->inited_proof_link()) { promise.set_value(td::Unit()); return; } - auto id = block_db_.get(); - auto P = td::PromiseCreator::lambda([id, handle, promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - handle->set_proof_link(); - td::actor::send_closure(id, &BlockDb::store_block_handle, std::move(handle), std::move(promise)); - } - }); - - td::actor::send_closure(file_db_, &FileDb::store_file, FileDb::RefId{fileref::ProofLink{handle->id()}}, proof->data(), - std::move(P)); + auto P = td::PromiseCreator::lambda( + [id = archive_db_.get(), handle, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + handle->set_proof_link(); + td::actor::send_closure(id, &ArchiveManager::update_handle, std::move(handle), std::move(promise)); + } + }); + td::actor::send_closure(archive_db_, &ArchiveManager::add_file, handle, fileref::ProofLink{handle->id()}, + proof->data(), std::move(P)); } void RootDb::get_block_proof_link(BlockHandle handle, td::Promise> promise) { @@ -189,13 +171,8 @@ void RootDb::get_block_proof_link(BlockHandle handle, td::Promisemoved_to_archive()) { - td::actor::send_closure(new_archive_db_, &ArchiveManager::read, handle->unix_time(), handle->is_key_block(), - FileDb::RefId{fileref::ProofLink{handle->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle->moved_to_storage() ? old_archive_db_.get() : file_db_.get(), &FileDb::load_file, - FileDb::RefId{fileref::ProofLink{handle->id()}}, std::move(P)); - } + td::actor::send_closure(archive_db_, &ArchiveManager::get_file, handle, fileref::ProofLink{handle->id()}, + std::move(P)); } } @@ -204,16 +181,16 @@ void RootDb::store_block_candidate(BlockCandidate candidate, td::Promise R) mutable { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { promise.set_value(td::Unit()); } }); - td::actor::send_closure(file_db_, &FileDb::store_file, - FileDb::RefId{fileref::Candidate{PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, - candidate.id, candidate.collated_file_hash}}, + td::actor::send_closure(archive_db_, &ArchiveManager::add_temp_file_short, + fileref::Candidate{PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.id, + candidate.collated_file_hash}, std::move(obj), std::move(P)); } @@ -234,18 +211,18 @@ void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash colla std::move(val->collated_data_)}); } }); - td::actor::send_closure(file_db_, &FileDb::load_file, - FileDb::RefId{fileref::Candidate{source, id, collated_data_file_hash}}, std::move(P)); + td::actor::send_closure(archive_db_, &ArchiveManager::get_temp_file_short, + fileref::Candidate{source, id, collated_data_file_hash}, std::move(P)); } void RootDb::store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) { - if (handle->moved_to_storage() || handle->moved_to_archive()) { + if (handle->moved_to_archive()) { promise.set_value(std::move(state)); return; } if (!handle->inited_state_boc()) { - auto P = td::PromiseCreator::lambda([b = block_db_.get(), root_hash = state->root_hash(), handle, + auto P = td::PromiseCreator::lambda([b = archive_db_.get(), root_hash = state->root_hash(), handle, promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); @@ -262,7 +239,7 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, promise.set_value(std::move(state)); }); - td::actor::send_closure(b, &BlockDb::store_block_handle, std::move(handle), std::move(P)); + td::actor::send_closure(b, &ArchiveManager::update_handle, std::move(handle), std::move(P)); } }); td::actor::send_closure(cell_db_, &CellDb::store_cell, handle->id(), state->root_cell(), std::move(P)); @@ -295,83 +272,46 @@ void RootDb::get_block_state(BlockHandle handle, td::Promise void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) { - auto id = block_db_.get(); - - auto P = td::PromiseCreator::lambda([id, promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_value(td::Unit()); - } - }); - - td::actor::send_closure(old_archive_db_, &FileDb::store_file, - FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, std::move(state), - std::move(P)); + td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id, + std::move(state), std::move(promise)); } void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { - td::actor::send_closure(old_archive_db_, &FileDb::load_file, - FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, std::move(promise)); + td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id, + std::move(promise)); } void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_size, td::Promise promise) { - td::actor::send_closure(old_archive_db_, &FileDb::load_file_slice, - FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, offset, max_size, - std::move(promise)); + td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_slice, block_id, masterchain_block_id, + offset, max_size, std::move(promise)); } void RootDb::check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { - td::actor::send_closure(old_archive_db_, &FileDb::check_file, - FileDb::RefId{fileref::PersistentState{block_id, masterchain_block_id}}, std::move(promise)); + td::actor::send_closure(archive_db_, &ArchiveManager::check_persistent_state, block_id, masterchain_block_id, + std::move(promise)); } void RootDb::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) { - auto id = block_db_.get(); - - auto P = td::PromiseCreator::lambda([id, promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_value(td::Unit()); - } - }); - - td::actor::send_closure(old_archive_db_, &FileDb::store_file, FileDb::RefId{fileref::ZeroState{block_id}}, - std::move(state), std::move(P)); + td::actor::send_closure(archive_db_, &ArchiveManager::add_zero_state, block_id, std::move(state), std::move(promise)); } void RootDb::get_zero_state_file(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(old_archive_db_, &FileDb::load_file, FileDb::RefId{fileref::ZeroState{block_id}}, - std::move(promise)); + td::actor::send_closure(archive_db_, &ArchiveManager::get_zero_state, block_id, std::move(promise)); } void RootDb::check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(old_archive_db_, &FileDb::check_file, FileDb::RefId{fileref::ZeroState{block_id}}, - std::move(promise)); + td::actor::send_closure(archive_db_, &ArchiveManager::check_zero_state, block_id, std::move(promise)); } void RootDb::store_block_handle(BlockHandle handle, td::Promise promise) { - if (handle->moved_to_archive()) { - td::actor::send_closure(new_archive_db_, &ArchiveManager::write_handle, std::move(handle), std::move(promise)); - } else { - td::actor::send_closure(block_db_, &BlockDb::store_block_handle, std::move(handle), std::move(promise)); - } + td::actor::send_closure(archive_db_, &ArchiveManager::update_handle, std::move(handle), std::move(promise)); } void RootDb::get_block_handle(BlockIdExt id, td::Promise promise) { - auto P = td::PromiseCreator::lambda( - [db = block_db_.get(), id, promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - td::actor::send_closure(db, &BlockDb::get_block_handle, id, std::move(promise)); - } else { - promise.set_value(R.move_as_ok()); - } - }); - td::actor::send_closure(new_archive_db_, &ArchiveManager::read_handle, id, std::move(P)); + td::actor::send_closure(archive_db_, &ArchiveManager::get_handle, id, std::move(promise)); } void RootDb::try_get_static_file(FileHash file_hash, td::Promise promise) { @@ -379,24 +319,20 @@ void RootDb::try_get_static_file(FileHash file_hash, td::Promise promise) { - if (handle->id().id.seqno == 0) { - promise.set_value(td::Unit()); - } else { - td::actor::send_closure(lt_db_, &LtDb::add_new_block, handle->id(), handle->logical_time(), handle->unix_time(), - std::move(promise)); - } + td::actor::create_actor("archiver", std::move(handle), archive_db_.get(), std::move(promise)) + .release(); } -void RootDb::get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) { - td::actor::send_closure(lt_db_, &LtDb::get_block_by_lt, account, lt, std::move(promise)); +void RootDb::get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_block_by_lt, account, lt, std::move(promise)); } -void RootDb::get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise promise) { - td::actor::send_closure(lt_db_, &LtDb::get_block_by_unix_time, account, ts, std::move(promise)); +void RootDb::get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_block_by_unix_time, account, ts, std::move(promise)); } -void RootDb::get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) { - td::actor::send_closure(lt_db_, &LtDb::get_block_by_seqno, account, seqno, std::move(promise)); +void RootDb::get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_block_by_seqno, account, seqno, std::move(promise)); } void RootDb::update_init_masterchain_block(BlockIdExt block, td::Promise promise) { @@ -451,19 +387,13 @@ void RootDb::get_hardforks(td::Promise> promise) { void RootDb::start_up() { cell_db_ = td::actor::create_actor("celldb", actor_id(this), root_path_ + "/celldb/"); - block_db_ = td::actor::create_actor("blockdb", actor_id(this), root_path_ + "/blockdb/"); - file_db_ = td::actor::create_actor("filedb", actor_id(this), root_path_ + "/files/", depth_, false); - old_archive_db_ = - td::actor::create_actor("filedbarchive", actor_id(this), root_path_ + "/archive/", depth_, true); - lt_db_ = td::actor::create_actor("ltdb", actor_id(this), root_path_ + "/ltdb/"); state_db_ = td::actor::create_actor("statedb", actor_id(this), root_path_ + "/state/"); static_files_db_ = td::actor::create_actor("staticfilesdb", actor_id(this), root_path_ + "/static/"); - new_archive_db_ = td::actor::create_actor("archivemanager", root_path_ + "/archive/"); + archive_db_ = td::actor::create_actor("archive", actor_id(this), root_path_); } -void RootDb::archive(BlockIdExt block_id, td::Promise promise) { - td::actor::create_actor("archiveblock", block_id, actor_id(this), file_db_.get(), - old_archive_db_.get(), new_archive_db_.get(), std::move(promise)) +void RootDb::archive(BlockHandle handle, td::Promise promise) { + td::actor::create_actor("archiveblock", std::move(handle), archive_db_.get(), std::move(promise)) .release(); } @@ -475,57 +405,86 @@ void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise promise) { td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_info_gc, block_id, std::move(promise)); } -void RootDb::allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise promise) { - ref_id.visit( - td::overloaded([&](const fileref::Empty &key) { UNREACHABLE(); }, - [&](const fileref::Block &key) { - td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_data_gc, key.block_id, - is_archive, std::move(promise)); - }, - [&](const fileref::ZeroState &key) { - td::actor::send_closure(validator_manager_, &ValidatorManager::allow_zero_state_file_gc, - key.block_id, std::move(promise)); - }, - [&](const fileref::PersistentState &key) { - CHECK(is_archive); - td::actor::send_closure(validator_manager_, &ValidatorManager::allow_persistent_state_file_gc, - key.block_id, key.masterchain_block_id, std::move(promise)); - }, - [&](const fileref::Proof &key) { - td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_proof_gc, - key.block_id, is_archive, std::move(promise)); - }, - [&](const fileref::ProofLink &key) { - td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_proof_link_gc, - key.block_id, is_archive, std::move(promise)); - }, - [&](const fileref::Signatures &key) { - CHECK(!is_archive); - td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_signatures_gc, - key.block_id, std::move(promise)); - }, - [&](const fileref::Candidate &key) { - CHECK(!is_archive); - td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_candidate_gc, - key.block_id, std::move(promise)); - }, - [&](const fileref::BlockInfo &key) { UNREACHABLE(); })); -} - void RootDb::prepare_stats(td::Promise>> promise) { auto merger = StatsMerger::create(std::move(promise)); - - td::actor::send_closure(file_db_, &FileDb::prepare_stats, merger.make_promise("filedb.")); - td::actor::send_closure(old_archive_db_, &FileDb::prepare_stats, merger.make_promise("archivedb.")); } void RootDb::truncate(td::Ref state, td::Promise promise) { td::MultiPromise mp; auto ig = mp.init_guard(); ig.add_promise(std::move(promise)); +} - td::actor::send_closure(lt_db_, &LtDb::truncate, state, ig.get_promise()); - td::actor::send_closure(block_db_, &BlockDb::truncate, state, ig.get_promise()); +void RootDb::add_key_block_proof(td::Ref proof, td::Promise promise) { + auto i = proof->get_basic_header_info().move_as_ok(); + td::actor::send_closure(archive_db_, &ArchiveManager::add_key_block_proof, i.utime, proof->block_id().seqno(), + i.end_lt, fileref::Proof{proof->block_id()}, proof->data(), std::move(promise)); +} + +void RootDb::add_key_block_proof_link(td::Ref proof, td::Promise promise) { + auto i = proof->get_basic_header_info().move_as_ok(); + td::actor::send_closure(archive_db_, &ArchiveManager::add_key_block_proof, i.utime, proof->block_id().seqno(), + i.end_lt, fileref::ProofLink{proof->block_id()}, proof->data(), std::move(promise)); +} +void RootDb::get_key_block_proof(BlockIdExt block_id, td::Promise> promise) { + auto P = td::PromiseCreator::lambda([block_id, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + promise.set_result(create_proof(block_id, R.move_as_ok())); + } + }); + td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::Proof{block_id}, std::move(P)); +} +void RootDb::get_key_block_proof_link(BlockIdExt block_id, td::Promise> promise) { + auto P = td::PromiseCreator::lambda([block_id, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + promise.set_result(create_proof_link(block_id, R.move_as_ok())); + } + }); + td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::Proof{block_id}, std::move(P)); +} + +void RootDb::check_key_block_proof_exists(BlockIdExt block_id, td::Promise promise) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_result(false); + } else { + promise.set_result(true); + } + }); + td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::Proof{block_id}, std::move(P)); +} +void RootDb::check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise promise) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_result(false); + } else { + promise.set_result(true); + } + }); + td::actor::send_closure(archive_db_, &ArchiveManager::get_key_block_proof, fileref::ProofLink{block_id}, + std::move(P)); +} + +void RootDb::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_id, masterchain_seqno, std::move(promise)); +} + +void RootDb::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, + td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_slice, archive_id, offset, limit, + std::move(promise)); +} + +void RootDb::set_async_mode(bool mode, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise)); +} + +void RootDb::run_gc(UnixTime ts) { + td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, ts); } } // namespace validator diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 47cd31be..373941e7 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -22,13 +22,10 @@ #include "td/db/KeyValueAsync.h" #include "ton/ton-types.h" -#include "blockdb.hpp" #include "celldb.hpp" -#include "filedb.hpp" -#include "ltdb.hpp" #include "statedb.hpp" #include "staticfilesdb.hpp" -#include "archive-db.hpp" +#include "archive-manager.hpp" namespace ton { @@ -85,9 +82,9 @@ class RootDb : public Db { void try_get_static_file(FileHash file_hash, td::Promise promise) override; void apply_block(BlockHandle handle, td::Promise promise) override; - void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) override; - void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise promise) override; - void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) override; + void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) override; + void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise promise) override; + void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) override; void update_init_masterchain_block(BlockIdExt block, td::Promise promise) override; void get_init_masterchain_block(td::Promise promise) override; @@ -108,16 +105,30 @@ class RootDb : public Db { void update_hardforks(std::vector blocks, td::Promise promise) override; void get_hardforks(td::Promise> promise) override; - void archive(BlockIdExt block_id, td::Promise promise) override; + void archive(BlockHandle handle, td::Promise promise) override; void allow_state_gc(BlockIdExt block_id, td::Promise promise); void allow_block_gc(BlockIdExt block_id, td::Promise promise); - void allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise promise); + //void allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise promise); void prepare_stats(td::Promise>> promise) override; void truncate(td::Ref state, td::Promise promise) override; + void add_key_block_proof(td::Ref proof, td::Promise promise) override; + void add_key_block_proof_link(td::Ref proof_link, td::Promise promise) override; + void get_key_block_proof(BlockIdExt block_id, td::Promise> promise) override; + void get_key_block_proof_link(BlockIdExt block_id, td::Promise> promise) override; + void check_key_block_proof_exists(BlockIdExt block_id, td::Promise promise) override; + void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise promise) override; + + void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override; + void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, + td::Promise promise) override; + void set_async_mode(bool mode, td::Promise promise) override; + + void run_gc(UnixTime ts) override; + private: td::actor::ActorId validator_manager_; @@ -125,13 +136,9 @@ class RootDb : public Db { td::uint32 depth_; td::actor::ActorOwn cell_db_; - td::actor::ActorOwn block_db_; - td::actor::ActorOwn file_db_; - td::actor::ActorOwn old_archive_db_; - td::actor::ActorOwn lt_db_; td::actor::ActorOwn state_db_; td::actor::ActorOwn static_files_db_; - td::actor::ActorOwn new_archive_db_; + td::actor::ActorOwn archive_db_; }; } // namespace validator diff --git a/validator/db/statedb.cpp b/validator/db/statedb.cpp index ed78626e..7add44e3 100644 --- a/validator/db/statedb.cpp +++ b/validator/db/statedb.cpp @@ -222,6 +222,22 @@ StateDb::StateDb(td::actor::ActorId root_db, std::string db_path) : root void StateDb::start_up() { kv_ = std::make_shared(td::RocksDb::open(db_path_).move_as_ok()); + + std::string value; + auto R = kv_->get(create_serialize_tl_object(), value); + R.ensure(); + if (R.move_as_ok() == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(value, true); + F.ensure(); + auto f = F.move_as_ok(); + CHECK(f->version_ == 2); + } else { + kv_->begin_transaction().ensure(); + kv_->set(create_serialize_tl_object(), + create_serialize_tl_object(2)) + .ensure(); + kv_->commit_transaction().ensure(); + } } } // namespace validator diff --git a/validator/db/statedb.hpp b/validator/db/statedb.hpp index af5fde4f..7da435db 100644 --- a/validator/db/statedb.hpp +++ b/validator/db/statedb.hpp @@ -50,6 +50,9 @@ class StateDb : public td::actor::Actor { void update_hardforks(std::vector blocks, td::Promise promise); void get_hardforks(td::Promise> promise); + void update_db_version(td::uint32 version, td::Promise promise); + void get_db_version(td::Promise promise); + StateDb(td::actor::ActorId root_db, std::string path); void start_up() override; diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 8cbbf327..9e690fff 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -194,11 +194,13 @@ void DownloadShardState::written_shard_state(td::Ref state) { handle_->set_applied(); handle_->set_split(state_->before_split()); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { + CHECK(handle->handle_moved_to_archive()); + CHECK(handle->moved_to_archive()) R.ensure(); td::actor::send_closure(SelfId, &DownloadShardState::written_block_handle); }); - handle_->flush(manager_, handle_, std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::archive, handle_, std::move(P)); } void DownloadShardState::written_block_handle() { diff --git a/validator/fabric.h b/validator/fabric.h index 54ad09a9..41df565b 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -54,8 +54,9 @@ void run_fake_accept_block_query(BlockIdExt id, td::Ref data, std::ve td::Promise promise); void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, td::actor::ActorId manager, td::Promise promise); -void run_apply_block_query(BlockIdExt id, td::Ref block, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise); +void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, + td::actor::ActorId manager, td::Timestamp timeout, + td::Promise promise); void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool skip_check_signatures = false); void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index 96cb2576..a5eaefac 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -309,6 +309,26 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities())); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, + std::move(P)); +} + +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, + td::Promise promise) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_, + query.offset_, query.max_size_, std::move(promise)); +} + void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_slave_sendExtMessage &query, td::Promise promise) { td::actor::send_closure( diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index 0db425fc..00e6e1f7 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -72,6 +72,10 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_slave_sendExtMessage &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, + td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, + td::Promise promise); // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, // td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 4c6caa20..e82b0905 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -31,6 +31,7 @@ #include "net/download-state.hpp" #include "net/download-proof.hpp" #include "net/get-next-key-blocks.hpp" +#include "net/download-archive-slice.hpp" #include "td/utils/Random.h" @@ -146,6 +147,7 @@ void FullNodeShardImpl::got_next_block(td::Result R) { } void FullNodeShardImpl::get_next_block() { + //return; attempt_++; auto P = td::PromiseCreator::lambda([validator_manager = validator_manager_, attempt = attempt_, block_id = handle_->id(), SelfId = actor_id(this)](td::Result R) { @@ -450,6 +452,26 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities())); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, + std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, + td::Promise promise) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_, + query.offset_, query.max_size_, std::move(promise)); +} + void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { auto B = fetch_tl_object(std::move(query), true); @@ -637,6 +659,15 @@ void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp t .release(); } +void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, + td::Promise promise) { + auto &b = choose_neighbour(); + td::actor::create_actor( + "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), timeout, + validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) + .release(); +} + void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise promise) { CHECK(!handle_); handle_ = std::move(handle); @@ -741,7 +772,7 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ authorized_keys.emplace(key, overlay::Overlays::max_fec_broadcast_size()); } - rules_ = overlay::OverlayPrivacyRules{overlay::Overlays::max_simple_broadcast_size(), std::move(authorized_keys)}; + rules_ = overlay::OverlayPrivacyRules{1 << 14, std::move(authorized_keys)}; td::actor::send_closure(overlays_, &overlay::Overlays::set_privacy_rules, adnl_id_, overlay_id_, rules_); if (update_cert) { diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 4527e811..3487a867 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -55,6 +55,8 @@ class FullNodeShard : public td::actor::Actor { td::Promise promise) = 0; virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) = 0; + virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, + td::Promise promise) = 0; virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 1724b9a7..875c4b53 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -61,7 +61,7 @@ class FullNodeShardImpl : public FullNodeShard { return 1; } static constexpr td::uint32 proto_version() { - return 1; + return 2; } static constexpr td::uint64 proto_capabilities() { return 0; @@ -120,6 +120,10 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, + td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, + td::Promise promise); // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, // td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); @@ -148,6 +152,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise) override; void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override; + void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, + td::Promise promise) override; void set_handle(BlockHandle handle, td::Promise promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 92040469..b55b1008 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -230,6 +230,14 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou td::actor::send_closure(shard, &FullNodeShard::get_next_key_blocks, block_id, timeout, std::move(promise)); } +void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, + td::Promise promise) { + auto shard = get_shard(ShardIdFull{masterchainId}); + CHECK(!shard.empty()); + td::actor::send_closure(shard, &FullNodeShard::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, + std::move(promise)); +} + td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { while (shards_.count(shard) == 0) { if (shard.shard == shardIdAll) { @@ -392,6 +400,11 @@ void FullNodeImpl::start_up() { td::Promise> promise) override { td::actor::send_closure(id_, &FullNodeImpl::get_next_key_blocks, block_id, timeout, std::move(promise)); } + void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, + td::Promise promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, + std::move(promise)); + } void new_key_block(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 8d08f6cd..e4eab04a 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -64,6 +64,8 @@ class FullNodeImpl : public FullNode { void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); + void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, + td::Promise promise); void got_key_block_proof(td::Ref proof); void got_zero_block_state(td::Ref state); diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index f7c5d277..18cc0472 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -41,14 +41,16 @@ using namespace std::literals::string_literals; AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - bool send_broadcast, td::actor::ActorId manager, - td::Promise promise) + td::Ref approve_signatures, bool send_broadcast, + td::actor::ActorId manager, td::Promise promise) : id_(id) , data_(std::move(data)) , prev_(std::move(prev)) , validator_set_(std::move(validator_set)) , signatures_(std::move(signatures)) + , approve_signatures_(std::move(approve_signatures)) , is_fake_(false) + , is_fork_(false) , send_broadcast_(send_broadcast) , manager_(manager) , promise_(std::move(promise)) { @@ -66,6 +68,7 @@ AcceptBlockQuery::AcceptBlockQuery(AcceptBlockQuery::IsFake fake, BlockIdExt id, , prev_(std::move(prev)) , validator_set_(std::move(validator_set)) , is_fake_(true) + , is_fork_(false) , send_broadcast_(false) , manager_(manager) , promise_(std::move(promise)) { @@ -75,6 +78,74 @@ AcceptBlockQuery::AcceptBlockQuery(AcceptBlockQuery::IsFake fake, BlockIdExt id, CHECK(prev_.size() > 0); } +AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref data, + td::actor::ActorId manager, td::Promise promise) + : id_(id) + , data_(std::move(data)) + , is_fake_(true) + , is_fork_(true) + , send_broadcast_(false) + , manager_(manager) + , promise_(std::move(promise)) { + state_keep_old_hash_.clear(); + state_old_hash_.clear(); + state_hash_.clear(); +} + +bool AcceptBlockQuery::precheck_header() { + VLOG(VALIDATOR_DEBUG) << "precheck_header()"; + // 0. sanity check + CHECK(data_.not_null()); + block_root_ = data_->root_cell(); + if (data_->block_id() != id_) { + return fatal_error("incorrect block id in block data: "s + data_->block_id().to_str() + " instead of " + + id_.to_str()); + } + // 1. root hash and file hash check + RootHash blk_rhash{block_root_->get_hash().bits()}; + if (blk_rhash != id_.root_hash) { + return fatal_error("block root hash mismatch: expected "s + id_.root_hash.to_hex() + ", found " + + blk_rhash.to_hex()); + } + if (is_fake_ || is_fork_) { + FileHash blk_fhash; + td::sha256(data_->data().as_slice(), blk_fhash.as_slice()); + if (blk_fhash != id_.file_hash) { + return fatal_error("block file hash mismatch: expected "s + id_.file_hash.to_hex() + ", computed " + + blk_fhash.to_hex()); + } + } + // 2. check header fields + std::vector prev; + ton::BlockIdExt mc_blkid; + bool after_split; + auto res = block::unpack_block_prev_blk_try(block_root_, id_, prev, mc_blkid, after_split); + if (res.is_error()) { + return fatal_error("invalid block header in AcceptBlock: "s + res.to_string()); + } + if (is_fork_) { + prev_ = prev; + } else if (prev_ != prev) { + return fatal_error("invalid previous block reference(s) in block header"); + } + // 3. unpack header and check vert_seqno fields + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + if (!(tlb::unpack_cell(block_root_, blk) && tlb::unpack_cell(blk.info, info))) { + return fatal_error("cannot unpack block header"); + } + if (info.vert_seqno_incr && !is_fork_) { + return fatal_error("block header has vert_seqno_incr set in an ordinary AcceptBlock"); + } + if (!info.vert_seqno_incr && is_fork_) { + return fatal_error("fork block header has no vert_seqno_incr"); + } + if (is_fork_ && !info.key_block) { + return fatal_error("fork block is not a key block"); + } + return true; +} + bool AcceptBlockQuery::create_new_proof() { // 0. check block's root hash VLOG(VALIDATOR_DEBUG) << "create_new_proof() : start"; @@ -93,7 +164,7 @@ bool AcceptBlockQuery::create_new_proof() { block::CurrencyCollection fees; ShardIdFull shard; if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(blk.info, info) && !info.version && - block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && !info.vert_seq_no && + block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) && tlb::unpack_cell(std::move(blk.extra), extra) && block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) && (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) { @@ -292,6 +363,26 @@ void AcceptBlockQuery::start_up() { fatal_error("no real SignatureSet passed to AcceptBlockQuery"); return; } + if (!is_fake_ && is_fork_) { + fatal_error("a non-fake AcceptBlockQuery for a forced fork block"); + return; + } + if (!is_fork_ && !prev_.size()) { + fatal_error("no previous blocks passed to AcceptBlockQuery"); + return; + } + if (is_fork_ && !is_masterchain()) { + fatal_error("cannot accept a non-masterchain fork block"); + return; + } + if (is_fork_ && data_.is_null()) { + fatal_error("cannot accept a fork block without explicit data"); + return; + } + if (data_.not_null() && !precheck_header()) { + fatal_error("invalid block header in AcceptBlock"); + return; + } td::actor::send_closure( manager_, &ValidatorManager::get_block_handle, id_, true, [SelfId = actor_id(this)](td::Result R) { @@ -330,8 +421,8 @@ void AcceptBlockQuery::written_block_data() { if (is_fake_) { signatures_ = Ref(create_signature_set(std::vector{})); } - td::actor::send_closure(manager_, &ValidatorManager::set_block_signatures, handle_, - signatures_, [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(manager_, &ValidatorManager::set_block_signatures, handle_, signatures_, + [SelfId = actor_id(this)](td::Result R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::written_block_signatures); }); @@ -365,8 +456,8 @@ void AcceptBlockQuery::written_block_info() { td::actor::send_closure(manager_, &ValidatorManager::wait_prev_block_state, handle_, priority(), timeout_, std::move(P)); } else { - td::actor::send_closure(manager_, &ValidatorManager::wait_block_data, handle_, priority(), - timeout_, [SelfId = actor_id(this)](td::Result> R) { + td::actor::send_closure(manager_, &ValidatorManager::wait_block_data, handle_, priority(), timeout_, + [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_block_data, R.move_as_ok()); @@ -382,6 +473,10 @@ void AcceptBlockQuery::got_block_data(td::Ref data) { fatal_error("block data does not contain a root cell"); return; } + if (!precheck_header()) { + fatal_error("invalid block header in AcceptBlock"); + return; + } if (handle_->received()) { written_block_data(); } else { @@ -406,8 +501,8 @@ void AcceptBlockQuery::got_prev_state(td::Ref state) { handle_->set_split(state_->before_split()); - td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, - state_, [SelfId = actor_id(this)](td::Result> R) { + td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, state_, + [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::written_state, R.move_as_ok()); }); @@ -479,8 +574,8 @@ void AcceptBlockQuery::got_last_mc_block(std::pair, Bl if (last_mc_id_.id.seqno < mc_blkid_.id.seqno) { VLOG(VALIDATOR_DEBUG) << "shardchain block refers to newer masterchain block " << mc_blkid_.to_str() << ", trying to obtain it"; - td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(), - timeout_, [SelfId = actor_id(this)](td::Result> R) { + td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(), timeout_, + [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_mc_state, R.move_as_ok()); @@ -586,7 +681,7 @@ void AcceptBlockQuery::require_proof_link(BlockIdExt id) { CHECK(ton::ShardIdFull(id) == ton::ShardIdFull(id_)); CHECK(id.id.seqno == id_.id.seqno - 1 - proof_links_.size()); td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_proof_link_short, id, timeout_, - [ SelfId = actor_id(this), id ](td::Result> R) { + [SelfId = actor_id(this), id](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_proof_link, id, R.move_as_ok()); @@ -785,7 +880,7 @@ void AcceptBlockQuery::written_block_info_2() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::applied); }); - run_apply_block_query(handle_->id(), data_, manager_, timeout_, std::move(P)); + run_apply_block_query(handle_->id(), data_, handle_->id(), manager_, timeout_, std::move(P)); } else { applied(); } diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index 799a289b..a0c7e576 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -47,12 +47,16 @@ using td::Ref; class AcceptBlockQuery : public td::actor::Actor { public: struct IsFake {}; + struct ForceFork {}; AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, - td::Ref validator_set, td::Ref signatures, bool send_broadcast, + td::Ref validator_set, td::Ref signatures, + td::Ref approve_signatures, bool send_broadcast, td::actor::ActorId manager, td::Promise promise); AcceptBlockQuery(IsFake fake, BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::actor::ActorId manager, td::Promise promise); + AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref data, + td::actor::ActorId manager, td::Promise promise); private: static constexpr td::uint32 priority() { @@ -90,7 +94,9 @@ class AcceptBlockQuery : public td::actor::Actor { std::vector prev_; Ref validator_set_; Ref signatures_; + Ref approve_signatures_; bool is_fake_; + bool is_fork_; bool send_broadcast_; bool ancestors_split_{false}, is_key_block_{false}; td::Timestamp timeout_ = td::Timestamp::in(600.0); @@ -128,6 +134,7 @@ class AcceptBlockQuery : public td::actor::Actor { static bool check_send_error(td::actor::ActorId SelfId, td::Result& res) { return res.is_error() && check_send_error(std::move(SelfId), res.move_as_error()); } + bool precheck_header(); bool create_new_proof(); bool unpack_proof_link(BlockIdExt id, Ref proof); diff --git a/validator/impl/check-proof.cpp b/validator/impl/check-proof.cpp index e85055b6..aa6c2748 100644 --- a/validator/impl/check-proof.cpp +++ b/validator/impl/check-proof.cpp @@ -165,7 +165,7 @@ bool CheckProof::init_parse(bool is_aux) { block::gen::ExtBlkRef::Record mcref; // _ ExtBlkRef = BlkMasterInfo; ShardIdFull shard; if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(blk.info, info) && !info.version && - block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && !info.vert_seq_no && + block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) && block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) && (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) { diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index a4671a8b..1163089e 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -119,7 +119,8 @@ void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector< td::Ref approve_signatures, bool send_broadcast, td::actor::ActorId manager, td::Promise promise) { td::actor::create_actor("accept", id, std::move(data), prev, std::move(validator_set), - std::move(signatures), send_broadcast, manager, std::move(promise)) + std::move(signatures), std::move(approve_signatures), send_broadcast, + manager, std::move(promise)) .release(); } @@ -134,13 +135,16 @@ void run_fake_accept_block_query(BlockIdExt id, td::Ref data, std::ve void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, td::actor::ActorId manager, td::Promise promise) { - promise.set_error(td::Status::Error(ErrorCode::error, "not implemented")); + td::actor::create_actor("fork/accept", AcceptBlockQuery::ForceFork(), id, std::move(data), + std::move(manager), std::move(promise)) + .release(); } -void run_apply_block_query(BlockIdExt id, td::Ref block, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise) { - td::actor::create_actor(PSTRING() << "apply " << id, id, std::move(block), manager, timeout, - std::move(promise)) +void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, + td::actor::ActorId manager, td::Timestamp timeout, + td::Promise promise) { + td::actor::create_actor(PSTRING() << "apply " << id, id, std::move(block), masterchain_block_id, manager, + timeout, std::move(promise)) .release(); } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 23831af0..196397a9 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -192,7 +192,7 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { } td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [ Self = actor_id(this), mode ](td::Result, BlockIdExt>> res) { + [Self = actor_id(this), mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -230,7 +230,7 @@ void LiteQuery::perform_getBlock(BlockIdExt blkid) { return; } td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -256,7 +256,7 @@ void LiteQuery::perform_getBlockHeader(BlockIdExt blkid, int mode) { return; } td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [ Self = actor_id(this), blkid, mode ](td::Result> res) { + [Self = actor_id(this), blkid, mode](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -371,7 +371,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { } if (blkid.id.seqno) { td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -381,7 +381,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { }); } else { td::actor::send_closure_later(manager_, &ValidatorManager::get_zero_state, blkid, - [ Self = actor_id(this), blkid ](td::Result res) { + [Self = actor_id(this), blkid](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -440,7 +440,7 @@ bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); @@ -466,7 +466,7 @@ bool LiteQuery::request_mc_proof(BlockIdExt blkid, int mode) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_block_proof_from_db_short, blkid, - [ Self = actor_id(this), blkid, mode ](td::Result> res) { + [Self = actor_id(this), blkid, mode](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load proof for "s + blkid.to_str() + " : ")); @@ -488,7 +488,7 @@ bool LiteQuery::request_mc_block_state(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); @@ -519,7 +519,7 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); @@ -541,7 +541,7 @@ bool LiteQuery::request_block_data(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); @@ -563,7 +563,7 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_block_proof_link_from_db_short, blkid, - [ Self = actor_id(this), blkid ](td::Result> res) { + [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : ")); @@ -588,7 +588,7 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { ++pending_; td::actor::send_closure_later( manager_, &ValidatorManager::get_zero_state, blkid, - [ Self = actor_id(this), blkid ](td::Result res) { + [Self = actor_id(this), blkid](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : ")); @@ -632,7 +632,7 @@ void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, LOG(INFO) << "sending a get_top_masterchain_state_block query to manager"; td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [Self = actor_id(this)](td::Result, BlockIdExt>> res)->void { + [Self = actor_id(this)](td::Result, BlockIdExt>> res) -> void { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1067,14 +1067,14 @@ void LiteQuery::continue_getTransactions(unsigned remaining, bool exact) { << " " << trans_lt_; td::actor::send_closure_later( manager_, &ValidatorManager::get_block_by_lt_from_db, ton::extract_addr_prefix(acc_workchain_, acc_addr_), - trans_lt_, [ Self = actor_id(this), remaining, manager = manager_ ](td::Result res) { + trans_lt_, [Self = actor_id(this), remaining, manager = manager_](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, res.move_as_error(), ton::BlockIdExt{}); } else { - auto blkid = res.move_as_ok(); - LOG(DEBUG) << "requesting data for block " << blkid.to_str(); - td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self, blkid, remaining](td::Result> res) { + auto handle = res.move_as_ok(); + LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, + [Self, blkid = handle->id(), remaining](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, res.move_as_error(), blkid); @@ -1141,7 +1141,7 @@ void LiteQuery::perform_getShardInfo(BlockIdExt blkid, ShardIdFull shard, bool e void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector param_list) { LOG(INFO) << "started a getConfigParams(" << blkid.to_str() << ", " << mode << ", ) liteserver query"; - set_continuation([ this, mode, param_list = std::move(param_list) ]() mutable { + set_continuation([this, mode, param_list = std::move(param_list)]() mutable { continue_getConfigParams(mode, std::move(param_list)); }); request_mc_block_data_state(blkid); @@ -1294,14 +1294,14 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni LOG(INFO) << "performing a lookupBlock(" << blkid.to_str() << ", " << mode << ", " << lt << ", " << utime << ") query"; auto P = td::PromiseCreator::lambda( - [ Self = actor_id(this), manager = manager_, mode = (mode >> 4) ](td::Result res) { + [Self = actor_id(this), manager = manager_, mode = (mode >> 4)](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { - auto blkid = res.move_as_ok(); - LOG(DEBUG) << "requesting data for block " << blkid.to_str(); - td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self, blkid, mode](td::Result> res) { + auto handle = res.move_as_ok(); + LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, + [Self, blkid = handle->id(), mode](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1449,7 +1449,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, if (mode & 0x1000) { BlockIdExt bblk = (from.seqno() > to.seqno()) ? from : to; td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, bblk, - [ Self = actor_id(this), from, to, bblk, mode ](td::Result> res) { + [Self = actor_id(this), from, to, bblk, mode](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1461,7 +1461,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, } else { td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [ Self = actor_id(this), from, to, mode ](td::Result, BlockIdExt>> res) { + [Self = actor_id(this), from, to, mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1474,7 +1474,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, } else if (mode & 2) { td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [ Self = actor_id(this), from, mode ](td::Result, BlockIdExt>> res) { + [Self = actor_id(this), from, mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { @@ -1485,7 +1485,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, }); } else { td::actor::send_closure_later(manager_, &ton::validator::ValidatorManager::get_shard_client_state, false, - [ Self = actor_id(this), from, mode ](td::Result res) { + [Self = actor_id(this), from, mode](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { diff --git a/validator/impl/proof.cpp b/validator/impl/proof.cpp index 6270a8f9..2abff149 100644 --- a/validator/impl/proof.cpp +++ b/validator/impl/proof.cpp @@ -88,6 +88,7 @@ td::Result ProofLinkQ::get_basic_header_info() const } res.cc_seqno = info.gen_catchain_seqno; res.utime = info.gen_utime; + res.end_lt = info.end_lt; res.validator_set_hash = info.gen_validator_list_hash_short; res.prev_key_mc_seqno = info.prev_key_block_seqno; return res; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 2350d171..0f16a3b9 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -430,6 +430,7 @@ bool ValidateQuery::init_parse() { return reject_query("a non-masterchain block cannot be a key block"); } if (info.vert_seqno_incr) { + // what about non-masterchain blocks? return reject_query("new blocks cannot have vert_seqno_incr set"); } if (info.after_merge != after_merge_) { @@ -1620,12 +1621,13 @@ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block: << " has unchanged catchain seqno " << cc_seqno << ", but it must have been updated for all shards"); } - if (!cc_updated && !info.before_merge_ && old_before_merge && !workchain_created) { + bool bm_cleared = !info.before_merge_ && old_before_merge; + if (!cc_updated && bm_cleared && !workchain_created) { return reject_query(PSTRING() << "new shard configuration for shard " << shard.to_str() << " has unchanged catchain seqno " << cc_seqno << " while the before_merge bit has been cleared"); } - if (cc_updated && (!update_shard_cc_ || (!info.before_merge_ && old_before_merge))) { + if (cc_updated && !(update_shard_cc_ || bm_cleared)) { return reject_query(PSTRING() << "new shard configuration for shard " << shard.to_str() << " has increased catchain seqno " << cc_seqno << " without a good reason"); } diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp new file mode 100644 index 00000000..46fcc0ae --- /dev/null +++ b/validator/import-db-slice.cpp @@ -0,0 +1,354 @@ +#include "import-db-slice.hpp" +#include "validator/db/fileref.hpp" +#include "td/utils/overloaded.h" +#include "validator/fabric.h" +#include "td/actor/MultiPromise.h" +#include "common/checksum.h" +#include "td/utils/port/path.h" + +namespace ton { + +namespace validator { + +ArchiveImporter::ArchiveImporter(std::string path, td::Ref state, BlockSeqno shard_client_seqno, + td::Ref opts, td::actor::ActorId manager, + td::Promise> promise) + : path_(std::move(path)) + , state_(std::move(state)) + , shard_client_seqno_(shard_client_seqno) + , opts_(std::move(opts)) + , manager_(manager) + , promise_(std::move(promise)) { +} + +void ArchiveImporter::start_up() { + auto R = Package::open(path_, false, false); + if (R.is_error()) { + abort_query(R.move_as_error()); + return; + } + package_ = std::make_shared(R.move_as_ok()); + + bool fail = false; + package_->iterate([&](std::string filename, td::BufferSlice data, td::uint64 offset) -> bool { + auto F = FileReference::create(filename); + if (F.is_error()) { + abort_query(F.move_as_error()); + fail = true; + return false; + } + auto f = F.move_as_ok(); + + BlockIdExt b; + bool is_proof = false; + bool ignore = true; + + f.ref().visit(td::overloaded( + [&](const fileref::Proof &p) { + b = p.block_id; + ignore = !b.is_masterchain(); + is_proof = true; + }, + [&](const fileref::ProofLink &p) { + b = p.block_id; + ignore = b.is_masterchain(); + is_proof = true; + }, + [&](const fileref::Block &p) { + b = p.block_id; + ignore = false; + is_proof = false; + }, + [&](const auto &p) { ignore = true; })); + + if (!ignore) { + blocks_[b][is_proof ? 0 : 1] = offset; + if (b.is_masterchain()) { + masterchain_blocks_[b.seqno()] = b; + } + } + return true; + }); + + if (fail) { + return; + } + + if (masterchain_blocks_.size() == 0) { + abort_query(td::Status::Error(ErrorCode::notready, "archive does not contain any masterchain blocks")); + return; + } + + auto seqno = masterchain_blocks_.begin()->first; + + check_masterchain_block(seqno); +} + +void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { + auto it = masterchain_blocks_.find(seqno); + if (it == masterchain_blocks_.end()) { + if (seqno == 0) { + abort_query(td::Status::Error(ErrorCode::notready, "no new blocks")); + return; + } + checked_all_masterchain_blocks(seqno - 1); + return; + } + if (seqno < state_->get_block_id().seqno()) { + if (!state_->check_old_mc_block_id(it->second)) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id")); + return; + } + check_masterchain_block(seqno + 1); + } else if (seqno == state_->get_block_id().seqno()) { + if (state_->get_block_id() != it->second) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id")); + return; + } + check_masterchain_block(seqno + 1); + } else { + if (seqno != state_->get_block_id().seqno() + 1) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "hole in masterchain seqno")); + return; + } + auto it2 = blocks_.find(it->second); + CHECK(it2 != blocks_.end()); + + auto R1 = package_->read(it2->second[0]); + if (R1.is_error()) { + abort_query(R1.move_as_error()); + return; + } + + auto proofR = create_proof(it->second, std::move(R1.move_as_ok().second)); + if (proofR.is_error()) { + abort_query(proofR.move_as_error()); + return; + } + + auto R2 = package_->read(it2->second[1]); + if (R2.is_error()) { + abort_query(R2.move_as_error()); + return; + } + + if (sha256_bits256(R2.ok().second.as_slice()) != it->second.file_hash) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "bad block file hash")); + return; + } + auto dataR = create_block(it->second, std::move(R2.move_as_ok().second)); + if (dataR.is_error()) { + abort_query(dataR.move_as_error()); + return; + } + + auto proof = proofR.move_as_ok(); + auto data = dataR.move_as_ok(); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = state_->get_block_id(), + data](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); + return; + } + auto handle = R.move_as_ok(); + CHECK(!handle->merge_before()); + if (handle->one_prev(true) != id) { + td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, + td::Status::Error(ErrorCode::protoviolation, "prev block mismatch")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporter::checked_masterchain_proof, std::move(handle), std::move(data)); + }); + + run_check_proof_query(it->second, std::move(proof), manager_, td::Timestamp::in(2.0), std::move(P), state_, + opts_->is_hardfork(it->second)); + } +} + +void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Ref data) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveImporter::applied_masterchain_block, std::move(handle)); + }); + run_apply_block_query(handle->id(), std::move(data), handle->id(), manager_, td::Timestamp::in(10.0), std::move(P)); +} + +void ArchiveImporter::applied_masterchain_block(BlockHandle handle) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveImporter::got_new_materchain_state, + td::Ref(R.move_as_ok())); + }); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P)); +} + +void ArchiveImporter::got_new_materchain_state(td::Ref state) { + state_ = std::move(state); + check_masterchain_block(state_->get_block_id().seqno() + 1); +} + +void ArchiveImporter::checked_all_masterchain_blocks(BlockSeqno seqno) { + max_shard_client_seqno_ = seqno; + check_next_shard_client_seqno(shard_client_seqno_ + 1); +} + +void ArchiveImporter::check_next_shard_client_seqno(BlockSeqno seqno) { + if (seqno > max_shard_client_seqno_) { + finish_query(); + return; + } + + if (seqno == max_shard_client_seqno_) { + got_masterchain_state(state_); + } else { + BlockIdExt b; + bool f = state_->get_old_mc_block_id(seqno, b); + CHECK(f); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveImporter::got_masterchain_state, + td::Ref{R.move_as_ok()}); + }); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, b, std::move(P)); + } +} + +void ArchiveImporter::got_masterchain_state(td::Ref state) { + auto s = state->get_shards(); + + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), seqno = state->get_block_id().seqno()](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporter::check_next_shard_client_seqno, seqno + 1); + } + }); + + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(P)); + + for (auto &shard : s) { + apply_shard_block(shard->top_block_id(), state->get_block_id(), ig.get_promise()); + } +} + +void ArchiveImporter::apply_shard_block(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), masterchain_block_id, promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveImporter::apply_shard_block_cont1, R.move_as_ok(), masterchain_block_id, + std::move(promise)); + }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, true, std::move(P)); +} + +void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt masterchain_block_id, + td::Promise promise) { + if (handle->is_applied()) { + promise.set_value(td::Unit()); + return; + } + + auto it = blocks_.find(handle->id()); + if (it == blocks_.end()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "no proof for shard block")); + return; + } + TRY_RESULT_PROMISE(promise, data, package_->read(it->second[0])); + TRY_RESULT_PROMISE(promise, proof, create_proof_link(handle->id(), std::move(data.second))); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle, masterchain_block_id, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporter::apply_shard_block_cont2, std::move(handle), + masterchain_block_id, std::move(promise)); + } + }); + run_check_proof_link_query(handle->id(), std::move(proof), manager_, td::Timestamp::in(10.0), std::move(P)); +} + +void ArchiveImporter::apply_shard_block_cont2(BlockHandle handle, BlockIdExt masterchain_block_id, + td::Promise promise) { + if (handle->is_applied()) { + promise.set_value(td::Unit()); + return; + } + CHECK(handle->id().seqno() > 0); + + if (!handle->merge_before() && handle->one_prev(true).shard_full() == handle->id().shard_full()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle, masterchain_block_id, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporter::apply_shard_block_cont3, std::move(handle), + masterchain_block_id, std::move(promise)); + } + }); + apply_shard_block(handle->one_prev(true), masterchain_block_id, std::move(P)); + } else { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + check_shard_block_applied(handle->one_prev(true), ig.get_promise()); + if (handle->merge_before()) { + check_shard_block_applied(handle->one_prev(false), ig.get_promise()); + } + } +} + +void ArchiveImporter::apply_shard_block_cont3(BlockHandle handle, BlockIdExt masterchain_block_id, + td::Promise promise) { + auto it = blocks_.find(handle->id()); + CHECK(it != blocks_.end()); + TRY_RESULT_PROMISE(promise, data, package_->read(it->second[1])); + if (sha256_bits256(data.second.as_slice()) != handle->id().file_hash) { + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad block file hash")); + return; + } + TRY_RESULT_PROMISE(promise, block, create_block(handle->id(), std::move(data.second))); + + run_apply_block_query(handle->id(), std::move(block), masterchain_block_id, manager_, td::Timestamp::in(10.0), + std::move(promise)); +} + +void ArchiveImporter::check_shard_block_applied(BlockIdExt block_id, td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto handle = R.move_as_ok(); + if (!handle->is_applied()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not applied")); + } else { + promise.set_value(td::Unit()); + } + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, false, std::move(P)); +} + +void ArchiveImporter::abort_query(td::Status error) { + if (promise_) { + promise_.set_error(std::move(error)); + td::unlink(path_).ensure(); + } + stop(); +} +void ArchiveImporter::finish_query() { + if (promise_) { + promise_.set_value(std::vector(state_->get_block_id().seqno(), max_shard_client_seqno_)); + td::unlink(path_).ensure(); + } + stop(); +} + +} // namespace validator + +} // namespace ton diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp new file mode 100644 index 00000000..5b7b1f1d --- /dev/null +++ b/validator/import-db-slice.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include "td/actor/actor.h" +#include "validator/interfaces/validator-manager.h" +#include "validator/db/package.hpp" + +namespace ton { + +namespace validator { + +class ArchiveImporter : public td::actor::Actor { + public: + ArchiveImporter(std::string path, td::Ref state, BlockSeqno shard_client_seqno, + td::Ref opts, td::actor::ActorId manager, + td::Promise> promise); + void start_up() override; + + void abort_query(td::Status error); + void finish_query(); + + void check_masterchain_block(BlockSeqno seqno); + void checked_masterchain_proof(BlockHandle handle, td::Ref data); + void applied_masterchain_block(BlockHandle handle); + void got_new_materchain_state(td::Ref state); + void checked_all_masterchain_blocks(BlockSeqno seqno); + + void check_next_shard_client_seqno(BlockSeqno seqno); + void got_masterchain_state(td::Ref state); + void apply_shard_block(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); + void apply_shard_block_cont1(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise promise); + void apply_shard_block_cont2(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise promise); + void apply_shard_block_cont3(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise promise); + void check_shard_block_applied(BlockIdExt block_id, td::Promise promise); + + private: + std::string path_; + td::Ref state_; + BlockSeqno shard_client_seqno_; + BlockSeqno max_shard_client_seqno_; + + td::Ref opts_; + + std::shared_ptr package_; + + td::actor::ActorId manager_; + td::Promise> promise_; + + std::map masterchain_blocks_; + std::map> blocks_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/interfaces/block-handle.h b/validator/interfaces/block-handle.h index b30269cf..00ecd8ab 100644 --- a/validator/interfaces/block-handle.h +++ b/validator/interfaces/block-handle.h @@ -32,8 +32,8 @@ struct BlockHandleInterface { public: virtual BlockIdExt id() const = 0; virtual bool received() const = 0; - virtual bool moved_to_storage() const = 0; virtual bool moved_to_archive() const = 0; + virtual bool handle_moved_to_archive() const = 0; virtual bool deleted() const = 0; virtual bool inited_next_left() const = 0; virtual bool inited_next_right() const = 0; @@ -49,6 +49,7 @@ struct BlockHandleInterface { virtual bool inited_split_after() const = 0; virtual bool inited_merge_before() const = 0; virtual bool inited_is_key_block() const = 0; + virtual bool inited_masterchain_ref_block() const = 0; virtual bool split_after() const = 0; virtual bool merge_before() const = 0; virtual bool is_key_block() const = 0; @@ -60,6 +61,7 @@ struct BlockHandleInterface { virtual bool is_zero() const = 0; virtual bool is_archived() const = 0; virtual bool is_applied() const = 0; + virtual BlockSeqno masterchain_ref_block() const = 0; virtual std::vector prev() const = 0; virtual BlockIdExt one_prev(bool left) const = 0; virtual std::vector next() const = 0; @@ -83,8 +85,8 @@ struct BlockHandleInterface { virtual void set_next(BlockIdExt next) = 0; virtual void set_prev(BlockIdExt prev) = 0; virtual void set_received() = 0; - virtual void set_moved_to_storage() = 0; virtual void set_moved_to_archive() = 0; + virtual void set_handle_moved_to_archive() = 0; virtual void set_deleted() = 0; virtual void set_split(bool value) = 0; virtual void set_merge(bool value) = 0; @@ -94,6 +96,7 @@ struct BlockHandleInterface { virtual void set_deleted_state_boc() = 0; virtual void set_archived() = 0; virtual void set_applied() = 0; + virtual void set_masterchain_ref_block(BlockSeqno seqno) = 0; virtual void unsafe_clear_applied() = 0; virtual void unsafe_clear_next() = 0; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 846371a1..632b0a9c 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -72,9 +72,9 @@ class Db : public td::actor::Actor { virtual void get_block_handle(BlockIdExt id, td::Promise promise) = 0; virtual void apply_block(BlockHandle handle, td::Promise promise) = 0; - virtual void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) = 0; - virtual void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise promise) = 0; - virtual void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) = 0; + virtual void get_block_by_lt(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) = 0; + virtual void get_block_by_unix_time(AccountIdPrefixFull account, UnixTime ts, td::Promise promise) = 0; + virtual void get_block_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) = 0; virtual void update_init_masterchain_block(BlockIdExt block, td::Promise promise) = 0; virtual void get_init_masterchain_block(td::Promise promise) = 0; @@ -95,11 +95,25 @@ class Db : public td::actor::Actor { virtual void update_hardforks(std::vector blocks, td::Promise promise) = 0; virtual void get_hardforks(td::Promise> promise) = 0; - virtual void archive(BlockIdExt block_id, td::Promise promise) = 0; + virtual void archive(BlockHandle handle, td::Promise promise) = 0; virtual void prepare_stats(td::Promise>> promise) = 0; virtual void truncate(td::Ref state, td::Promise promise) = 0; + + virtual void add_key_block_proof(td::Ref proof, td::Promise promise) = 0; + virtual void add_key_block_proof_link(td::Ref proof_link, td::Promise promise) = 0; + virtual void get_key_block_proof(BlockIdExt block_id, td::Promise> promise) = 0; + virtual void get_key_block_proof_link(BlockIdExt block_id, td::Promise> promise) = 0; + virtual void check_key_block_proof_exists(BlockIdExt block_id, td::Promise promise) = 0; + virtual void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise promise) = 0; + + virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; + virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, + td::Promise promise) = 0; + virtual void set_async_mode(bool mode, td::Promise promise) = 0; + + virtual void run_gc(UnixTime ts) = 0; }; } // namespace validator diff --git a/validator/interfaces/proof.h b/validator/interfaces/proof.h index 70598e68..5830f467 100644 --- a/validator/interfaces/proof.h +++ b/validator/interfaces/proof.h @@ -29,6 +29,7 @@ class ProofLink : public td::CntObject { public: struct BasicHeaderInfo { UnixTime utime; + LogicalTime end_lt; CatchainSeqno cc_seqno; td::uint32 validator_set_hash; BlockSeqno prev_key_mc_seqno; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 4328ce4f..71919d21 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -147,6 +147,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) = 0; virtual void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) = 0; + virtual void archive(BlockHandle handle, td::Promise promise) = 0; + virtual void check_is_hardfork(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_vertical_seqno(BlockSeqno seqno, td::Promise promise) = 0; diff --git a/validator/invariants.hpp b/validator/invariants.hpp index 01b114c6..d10495a7 100644 --- a/validator/invariants.hpp +++ b/validator/invariants.hpp @@ -45,7 +45,7 @@ class ValidatorInvariants { CHECK(handle->inited_merge_before()); CHECK(handle->inited_split_after()); CHECK(handle->inited_prev()); - CHECK(handle->inited_signatures()); + CHECK(handle->inited_signatures() || handle->is_applied()); CHECK(handle->inited_state_root_hash()); CHECK(handle->inited_logical_time()); CHECK(handle->inited_unix_time()); diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 9798bb17..7c9e2e6a 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -223,6 +223,42 @@ void ValidatorManagerImpl::get_block_proof(BlockHandle handle, td::Promise promise) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto B = R.move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P)); +} + +void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), block_id, db = db_.get()](td::Result> R) mutable { + if (R.is_error()) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto B = R.move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db, &Db::get_key_block_proof, block_id, std::move(P)); + } else { + auto B = R.move_as_ok()->export_as_proof_link().move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P)); +} + void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { auto R = create_ext_message(std::move(data)); if (R.is_ok()) { @@ -582,17 +618,17 @@ void ValidatorManagerImpl::get_block_proof_link_from_db_short(BlockIdExt block_i } void ValidatorManagerImpl::get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, - td::Promise promise) { + td::Promise promise) { td::actor::send_closure(db_, &Db::get_block_by_lt, account, lt, std::move(promise)); } void ValidatorManagerImpl::get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) { + td::Promise promise) { td::actor::send_closure(db_, &Db::get_block_by_unix_time, account, ts, std::move(promise)); } void ValidatorManagerImpl::get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) { + td::Promise promise) { td::actor::send_closure(db_, &Db::get_block_by_seqno, account, seqno, std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 6da9110e..dbb85b2c 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -119,6 +119,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_proof_link(BlockHandle block_id, td::Promise promise) override { UNREACHABLE(); } + void get_key_block_proof(BlockIdExt block_id, td::Promise promise) override; + void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; @@ -200,11 +202,11 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_proof_link_from_db(BlockHandle handle, td::Promise> promise) override; void get_block_proof_link_from_db_short(BlockIdExt id, td::Promise> promise) override; - void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) override; + void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) override; void get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) override; + td::Promise promise) override; void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) override; + td::Promise promise) override; // get block handle declared in parent class void write_handle(BlockHandle handle, td::Promise promise) override; @@ -259,6 +261,14 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); } + void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override { + UNREACHABLE(); + } + void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, + td::Promise promise) override { + UNREACHABLE(); + } + void add_shard_block_description(td::Ref desc); void register_block_handle(BlockHandle handle, td::Promise promise); @@ -327,6 +337,9 @@ class ValidatorManagerImpl : public ValidatorManager { void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override { promise.set_result(false); } + void archive(BlockHandle handle, td::Promise promise) override { + td::actor::send_closure(db_, &Db::archive, std::move(handle), std::move(promise)); + } void update_last_known_key_block(BlockHandle handle, bool send_request) override { } void update_shard_client_block_handle(BlockHandle handle, td::Promise promise) override { diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index d96eef77..d2a7b861 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -104,16 +104,22 @@ void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto proof_link = pp.move_as_ok(); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof_link](td::Result R) { if (R.is_error()) { LOG(WARNING) << "downloaded proof link failed: " << R.move_as_error(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); } else { - td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::try_download_key_blocks, false); + auto P = td::PromiseCreator::lambda([SelfId, handle = R.move_as_ok()](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::try_download_key_blocks, false); + }); + td::actor::send_closure(db, &Db::add_key_block_proof_link, proof_link, std::move(P)); } }); - run_check_proof_link_query(handle_->id(), pp.move_as_ok(), manager_, td::Timestamp::in(60.0), std::move(P)); + run_check_proof_link_query(handle_->id(), proof_link, manager_, td::Timestamp::in(60.0), std::move(P)); } void ValidatorManagerMasterchainReiniter::downloaded_zero_state() { @@ -259,6 +265,8 @@ void ValidatorManagerMasterchainReiniter::download_masterchain_state() { void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Ref state) { state_ = td::Ref{std::move(state)}; + CHECK(handle_->received_state()); + CHECK(handle_->is_applied()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); @@ -452,9 +460,9 @@ void ValidatorManagerMasterchainStarter::got_hardforks(std::vector v return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::got_truncate_block_id, R.move_as_ok()); + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::got_truncate_block_handle, R.move_as_ok()); }); td::actor::send_closure(db_, &Db::get_block_by_seqno, AccountIdPrefixFull{masterchainId, 0}, b.seqno() - 1, std::move(P)); diff --git a/validator/manager.cpp b/validator/manager.cpp index 2198d42a..4a627196 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -29,6 +29,7 @@ #include "ton/ton-io.hpp" #include "state-serializer.hpp" #include "get-next-key-blocks.h" +#include "import-db-slice.hpp" #include "auto/tl/lite_api.h" #include "tl-utils/lite-utils.hpp" @@ -189,6 +190,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise R) mutable { @@ -198,7 +200,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise promise) { @@ -324,6 +326,42 @@ void ValidatorManagerImpl::get_block_proof_link(BlockHandle handle, td::Promise< td::actor::send_closure(db_, &Db::get_block_proof_link, std::move(handle), std::move(P)); } +void ValidatorManagerImpl::get_key_block_proof(BlockIdExt block_id, td::Promise promise) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto B = R.move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P)); +} + +void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), block_id, db = db_.get()](td::Result> R) mutable { + if (R.is_error()) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto B = R.move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db, &Db::get_key_block_proof, block_id, std::move(P)); + } else { + auto B = R.move_as_ok()->export_as_proof_link().move_as_ok(); + promise.set_value(B->data()); + } + }); + + td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P)); +} + void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { if (!is_validator()) { return; @@ -899,17 +937,17 @@ void ValidatorManagerImpl::get_block_proof_link_from_db_short(BlockIdExt block_i } void ValidatorManagerImpl::get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, - td::Promise promise) { + td::Promise promise) { td::actor::send_closure(db_, &Db::get_block_by_lt, account, lt, std::move(promise)); } void ValidatorManagerImpl::get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) { + td::Promise promise) { td::actor::send_closure(db_, &Db::get_block_by_unix_time, account, ts, std::move(promise)); } void ValidatorManagerImpl::get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) { + td::Promise promise) { td::actor::send_closure(db_, &Db::get_block_by_seqno, account, seqno, std::move(promise)); } @@ -1339,6 +1377,7 @@ void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_, opts_->get_filedb_depth()); lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); token_manager_ = td::actor::create_actor("tokenmanager"); + td::mkdir(db_root_ + "/tmp/").ensure(); auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -1375,6 +1414,7 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { gc_masterchain_state_ = std::move(R.gc_state); shard_client_ = std::move(R.clients); + td::actor::send_closure(shard_client_, &ShardClient::start); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -1741,7 +1781,7 @@ void ValidatorManagerImpl::allow_persistent_state_file_gc(BlockIdExt block_id, B } void ValidatorManagerImpl::allow_archive(BlockIdExt block_id, td::Promise promise) { - if (!gc_masterchain_handle_) { + /*if (!gc_masterchain_handle_) { promise.set_result(false); return; } @@ -1784,7 +1824,8 @@ void ValidatorManagerImpl::allow_archive(BlockIdExt block_id, td::Promise promise.set_result(true); } }); - td::actor::send_closure(db_, &Db::archive, block_id, std::move(P)); + td::actor::send_closure(db_, &Db::archive, block_id, std::move(P));*/ + promise.set_result(false); } void ValidatorManagerImpl::allow_delete(BlockIdExt block_id, td::Promise promise) { @@ -1797,10 +1838,6 @@ void ValidatorManagerImpl::allow_delete(BlockIdExt block_id, td::Promise p return; } auto handle = R.move_as_ok(); - if (!handle->moved_to_storage()) { - promise.set_result(false); - return; - } if (!handle->inited_unix_time()) { promise.set_result(true); return; @@ -1843,15 +1880,13 @@ void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise } void ValidatorManagerImpl::allow_block_info_gc(BlockIdExt block_id, td::Promise promise) { - promise.set_result(false); - return; - /*auto P = + auto P = td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_result(false); } else { auto handle = R.move_as_ok(); - if (!handle->moved_to_archive()) { + if (!handle->moved_to_archive() || !handle->is_applied()) { promise.set_result(false); } else { auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { @@ -1862,7 +1897,7 @@ void ValidatorManagerImpl::allow_block_info_gc(BlockIdExt block_id, td::Promise< } } }); - get_block_handle(block_id, false, std::move(P));*/ + get_block_handle(block_id, false, std::move(P)); } void ValidatorManagerImpl::got_next_gc_masterchain_handle(BlockHandle handle) { @@ -1896,7 +1931,8 @@ void ValidatorManagerImpl::advance_gc(BlockHandle handle, td::Ref promise) { - auto seqno = handle->id().seqno(); + shard_client_handle_ = std::move(handle); + auto seqno = shard_client_handle_->id().seqno(); shard_client_update(seqno); promise.set_value(td::Unit()); } @@ -1928,15 +1964,26 @@ void ValidatorManagerImpl::state_serializer_update(BlockSeqno seqno) { void ValidatorManagerImpl::alarm() { try_advance_gc_masterchain_block(); alarm_timestamp() = td::Timestamp::in(1.0); + if (gc_masterchain_handle_) { + td::actor::send_closure(db_, &Db::run_gc, gc_masterchain_handle_->unix_time()); + } if (log_status_at_.is_in_past()) { if (last_masterchain_block_handle_) { LOG(INFO) << "STATUS: last_masterchain_block_ago=" << td::format::as_time(td::Clocks::system() - last_masterchain_block_handle_->unix_time()) << " last_known_key_block_ago=" - << td::format::as_time(td::Clocks::system() - last_known_key_block_handle_->unix_time()); + << td::format::as_time(td::Clocks::system() - last_known_key_block_handle_->unix_time()) + << " shard_client_ago=" + << td::format::as_time(td::Clocks::system() - + (shard_client_handle_ ? shard_client_handle_->unix_time() : 0)); } log_status_at_ = td::Timestamp::in(60.0); } + if (false && !downloading_archive_slice_ && shard_client_handle_ && + shard_client_handle_->unix_time() + 600 <= td::Clocks::system() && next_download_archive_slice_at_.is_in_past()) { + next_download_archive_slice_at_ = td::Timestamp::in(10.0); + try_download_archive_slice(); + } alarm_timestamp().relax(log_status_at_); if (resend_shard_blocks_at_ && resend_shard_blocks_at_.is_in_past()) { resend_shard_blocks_at_ = td::Timestamp::never(); @@ -1964,17 +2011,6 @@ void ValidatorManagerImpl::alarm() { if (check_shard_clients_.is_in_past()) { check_shard_clients_ = td::Timestamp::in(10.0); - if (!shard_client_.empty()) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - VLOG(VALIDATOR_WARNING) << "failed to get shard client status: " << R.move_as_error(); - } else { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::shard_client_update, R.move_as_ok()); - } - }); - td::actor::send_closure(shard_client_, &ShardClient::get_processed_masterchain_block, std::move(P)); - } - if (!serializer_.empty()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -2017,6 +2053,58 @@ void ValidatorManagerImpl::try_get_static_file(FileHash file_hash, td::Promise R) { + if (R.is_error()) { + LOG(INFO) << "failed to download archive slice: " << R.error(); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::failed_to_download_archive_slice); + } else { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::downloaded_archive_slice, R.move_as_ok()); + } + }); + callback_->download_archive(shard_client_handle_->id().seqno(), db_root_ + "/tmp/", td::Timestamp::in(3600.0), + std::move(P)); +} + +void ValidatorManagerImpl::failed_to_download_archive_slice() { + downloading_archive_slice_ = false; +} + +void ValidatorManagerImpl::downloaded_archive_slice(std::string name) { + LOG(INFO) << "downloaded archive slice: " << name; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + LOG(INFO) << "failed to check downloaded archive slice: " << R.error(); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::failed_to_download_archive_slice); + } else { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.move_as_ok()); + } + }); + + td::actor::create_actor("archiveimport", name, last_masterchain_state_, + shard_client_handle_->id().seqno(), opts_, actor_id(this), std::move(P)) + .release(); +} + +void ValidatorManagerImpl::checked_archive_slice(std::vector seqno) { + CHECK(seqno.size() == 2); + LOG(INFO) << "checked downloaded archive slice: mc_top_seqno=" << seqno[0] << " shard_top_seqno_=" << seqno[1]; + downloading_archive_slice_ = false; + next_download_archive_slice_at_ = td::Timestamp::in(10.0); +} + +void ValidatorManagerImpl::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_archive_id, masterchain_seqno, std::move(promise)); +} + +void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_archive_slice, archive_id, offset, limit, std::move(promise)); +} + bool ValidatorManagerImpl::is_validator() { return temp_keys_.size() > 0 || permanent_keys_.size() > 0; } diff --git a/validator/manager.hpp b/validator/manager.hpp index 669f1c66..2177766e 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -236,6 +236,7 @@ class ValidatorManagerImpl : public ValidatorManager { BlockHandle last_key_block_handle_; BlockHandle last_known_key_block_handle_; + BlockHandle shard_client_handle_; BlockHandle gc_masterchain_handle_; td::Ref gc_masterchain_state_; @@ -308,6 +309,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::int64 max_length, td::Promise promise) override; void get_block_proof(BlockHandle handle, td::Promise promise) override; void get_block_proof_link(BlockHandle block_id, td::Promise promise) override; + void get_key_block_proof(BlockIdExt block_id, td::Promise promise) override; + void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; @@ -385,11 +388,11 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_proof_link_from_db(BlockHandle handle, td::Promise> promise) override; void get_block_proof_link_from_db_short(BlockIdExt id, td::Promise> promise) override; - void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) override; + void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, td::Promise promise) override; void get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) override; + td::Promise promise) override; void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) override; + td::Promise promise) override; // get block handle declared in parent class void write_handle(BlockHandle handle, td::Promise promise) override; @@ -424,6 +427,10 @@ class ValidatorManagerImpl : public ValidatorManager { void get_async_serializer_state(td::Promise promise) override; void try_get_static_file(FileHash file_hash, td::Promise promise) override; + void try_download_archive_slice(); + void downloaded_archive_slice(std::string name); + void checked_archive_slice(std::vector seqno); + void failed_to_download_archive_slice(); void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override { @@ -431,6 +438,10 @@ class ValidatorManagerImpl : public ValidatorManager { std::move(promise)); } + void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override; + void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, + td::Promise promise) override; + void check_is_hardfork(BlockIdExt block_id, td::Promise promise) override { CHECK(block_id.is_masterchain()); promise.set_result(opts_->is_hardfork(block_id)); @@ -484,6 +495,9 @@ class ValidatorManagerImpl : public ValidatorManager { allow_block_state_gc(block_id, std::move(promise)); } void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override; + void archive(BlockHandle handle, td::Promise promise) override { + td::actor::send_closure(db_, &Db::archive, std::move(handle), std::move(promise)); + } void send_peek_key_block_request(); void got_next_key_blocks(std::vector vec); @@ -544,6 +558,9 @@ class ValidatorManagerImpl : public ValidatorManager { bool started_ = false; bool allow_validate_ = false; + bool downloading_archive_slice_ = false; + td::Timestamp next_download_archive_slice_at_ = td::Timestamp::now(); + private: double state_ttl() const { return opts_->state_ttl(); diff --git a/validator/net/download-archive-slice.cpp b/validator/net/download-archive-slice.cpp new file mode 100644 index 00000000..8b6e7188 --- /dev/null +++ b/validator/net/download-archive-slice.cpp @@ -0,0 +1,177 @@ +#include "download-archive-slice.hpp" +#include "td/utils/port/path.h" +#include "td/utils/overloaded.h" + +namespace ton { + +namespace validator { + +namespace fullnode { + +DownloadArchiveSlice::DownloadArchiveSlice( + BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, + overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, + td::actor::ActorId validator_manager, td::actor::ActorId rldp, + td::actor::ActorId overlays, td::actor::ActorId adnl, + td::actor::ActorId client, td::Promise promise) + : masterchain_seqno_(masterchain_seqno) + , tmp_dir_(std::move(tmp_dir)) + , local_id_(local_id) + , overlay_id_(overlay_id) + , download_from_(download_from) + , timeout_(timeout) + , validator_manager_(validator_manager) + , rldp_(rldp) + , overlays_(overlays) + , adnl_(adnl) + , client_(client) + , promise_(std::move(promise)) { +} + +void DownloadArchiveSlice::abort_query(td::Status reason) { + if (promise_) { + promise_.set_error(std::move(reason)); + if (!fd_.empty()) { + td::unlink(tmp_name_).ensure(); + fd_.close(); + } + } + stop(); +} + +void DownloadArchiveSlice::alarm() { + abort_query(td::Status::Error(ErrorCode::timeout, "timeout")); +} + +void DownloadArchiveSlice::finish_query() { + if (promise_) { + promise_.set_value(std::move(tmp_name_)); + fd_.close(); + } + stop(); +} + +void DownloadArchiveSlice::start_up() { + alarm_timestamp() = timeout_; + + auto R = td::mkstemp(tmp_dir_); + if (R.is_error()) { + abort_query(R.move_as_error_prefix("failed to open temp file: ")); + return; + } + auto r = R.move_as_ok(); + fd_ = std::move(r.first); + tmp_name_ = std::move(r.second); + + if (download_from_.is_zero() && client_.empty()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadArchiveSlice::abort_query, R.move_as_error()); + } else { + auto vec = R.move_as_ok(); + if (vec.size() == 0) { + td::actor::send_closure(SelfId, &DownloadArchiveSlice::abort_query, + td::Status::Error(ErrorCode::notready, "no nodes")); + } else { + td::actor::send_closure(SelfId, &DownloadArchiveSlice::got_node_to_download, vec[0]); + } + } + }); + + td::actor::send_closure(overlays_, &overlay::Overlays::get_overlay_random_peers, local_id_, overlay_id_, 1, + std::move(P)); + } else { + got_node_to_download(download_from_); + } +} + +void DownloadArchiveSlice::got_node_to_download(adnl::AdnlNodeIdShort download_from) { + download_from_ = download_from; + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadArchiveSlice::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadArchiveSlice::got_archive_info, R.move_as_ok()); + } + }); + + auto q = create_serialize_tl_object(masterchain_seqno_); + if (client_.empty()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_query, download_from_, local_id_, overlay_id_, + "get_archive_info", std::move(P), td::Timestamp::in(3.0), std::move(q)); + } else { + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_archive_info", + create_serialize_tl_object_suffix(std::move(q)), + td::Timestamp::in(1.0), std::move(P)); + } +} + +void DownloadArchiveSlice::got_archive_info(td::BufferSlice data) { + auto F = fetch_tl_object(std::move(data), true); + if (F.is_error()) { + abort_query(F.move_as_error_prefix("failed to parse ArchiveInfo answer")); + return; + } + auto f = F.move_as_ok(); + + bool fail = false; + ton_api::downcast_call(*f.get(), td::overloaded( + [&](const ton_api::tonNode_archiveNotFound &obj) { + abort_query(td::Status::Error(ErrorCode::notready, "remote db not found")); + fail = true; + }, + [&](const ton_api::tonNode_archiveInfo &obj) { archive_id_ = obj.id_; })); + if (fail) { + return; + } + + get_archive_slice(); +} + +void DownloadArchiveSlice::get_archive_slice() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadArchiveSlice::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadArchiveSlice::got_archive_slice, R.move_as_ok()); + } + }); + + auto q = create_serialize_tl_object(archive_id_, offset_, slice_size()); + if (client_.empty()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, + "get_archive_slice", std::move(P), td::Timestamp::in(3.0), std::move(q), + slice_size() + 1024, rldp_); + } else { + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_archive_slice", + create_serialize_tl_object_suffix(std::move(q)), + td::Timestamp::in(1.0), std::move(P)); + } +} + +void DownloadArchiveSlice::got_archive_slice(td::BufferSlice data) { + auto R = fd_.write(data.as_slice()); + if (R.is_error()) { + abort_query(R.move_as_error_prefix("failed to write temp file: ")); + return; + } + if (R.move_as_ok() != data.size()) { + abort_query(td::Status::Error(ErrorCode::error, "short write to temp file")); + return; + } + + offset_ += data.size(); + + if (data.size() < slice_size()) { + finish_query(); + } else { + get_archive_slice(); + } +} + +} // namespace fullnode + +} // namespace validator + +} // namespace ton diff --git a/validator/net/download-archive-slice.hpp b/validator/net/download-archive-slice.hpp new file mode 100644 index 00000000..018650ad --- /dev/null +++ b/validator/net/download-archive-slice.hpp @@ -0,0 +1,83 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#pragma once + +#include "overlay/overlays.h" +#include "ton/ton-types.h" +#include "validator/validator.h" +#include "rldp/rldp.h" +#include "adnl/adnl-ext-client.h" +#include "td/utils/port/FileFd.h" + +namespace ton { + +namespace validator { + +namespace fullnode { + +class DownloadArchiveSlice : public td::actor::Actor { + public: + DownloadArchiveSlice(BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, + overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, + td::actor::ActorId validator_manager, + td::actor::ActorId rldp, td::actor::ActorId overlays, + td::actor::ActorId adnl, td::actor::ActorId client, + td::Promise promise); + + void abort_query(td::Status reason); + void alarm() override; + void finish_query(); + + void start_up() override; + void got_node_to_download(adnl::AdnlNodeIdShort node); + void got_archive_info(td::BufferSlice data); + void get_archive_slice(); + void got_archive_slice(td::BufferSlice data); + + static constexpr td::uint32 slice_size() { + return 1 << 17; + } + + private: + BlockSeqno masterchain_seqno_; + std::string tmp_dir_; + std::string tmp_name_; + td::FileFd fd_; + adnl::AdnlNodeIdShort local_id_; + overlay::OverlayIdShort overlay_id_; + td::uint64 offset_ = 0; + td::uint64 archive_id_; + + adnl::AdnlNodeIdShort download_from_ = adnl::AdnlNodeIdShort::zero(); + + td::Timestamp timeout_; + td::actor::ActorId validator_manager_; + td::actor::ActorId rldp_; + td::actor::ActorId overlays_; + td::actor::ActorId adnl_; + td::actor::ActorId client_; + td::Promise promise_; +}; + +} // namespace fullnode + +} // namespace validator + +} // namespace ton + diff --git a/validator/net/download-proof.cpp b/validator/net/download-proof.cpp index 3fcfd6a9..b8513c9c 100644 --- a/validator/net/download-proof.cpp +++ b/validator/net/download-proof.cpp @@ -79,6 +79,33 @@ void DownloadProof::finish_query() { void DownloadProof::start_up() { alarm_timestamp() = timeout_; + if (!block_id_.is_masterchain()) { + checked_db(); + return; + } + + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), l = allow_partial_proof_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadProof::checked_db); + } else { + if (l) { + td::actor::send_closure(SelfId, &DownloadProof::got_block_partial_proof, R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &DownloadProof::got_block_proof, R.move_as_ok()); + } + } + }); + if (allow_partial_proof_) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof_link, block_id_, + std::move(P)); + } else { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof, block_id_, + std::move(P)); + } +} + +void DownloadProof::checked_db() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadProof::abort_query, diff --git a/validator/net/download-proof.hpp b/validator/net/download-proof.hpp index 64ed65bd..e72c6c2e 100644 --- a/validator/net/download-proof.hpp +++ b/validator/net/download-proof.hpp @@ -44,6 +44,7 @@ class DownloadProof : public td::actor::Actor { void finish_query(); void start_up() override; + void checked_db(); void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_proof_description(td::BufferSlice proof_description); diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 2f05bdac..a609dcf9 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -39,6 +39,15 @@ void ShardClient::start_up() { td::actor::send_closure(manager_, &ValidatorManager::get_shard_client_state, true, std::move(P)); } +void ShardClient::start() { + if (!started_ && masterchain_state_.not_null()) { + started_ = true; + apply_all_shards(); + } else { + started_ = true; + } +} + void ShardClient::got_state_from_db(BlockIdExt state) { CHECK(!init_mode_); @@ -133,7 +142,9 @@ void ShardClient::download_masterchain_state() { void ShardClient::got_masterchain_block_state(td::Ref state) { masterchain_state_ = std::move(state); build_shard_overlays(); - apply_all_shards(); + if (started_) { + apply_all_shards(); + } } void ShardClient::apply_all_shards() { @@ -170,8 +181,8 @@ void ShardClient::apply_all_shards() { } void ShardClient::downloaded_shard_state(td::Ref state, td::Promise promise) { - run_apply_block_query(state->get_block_id(), td::Ref{}, manager_, td::Timestamp::in(600), - std::move(promise)); + run_apply_block_query(state->get_block_id(), td::Ref{}, masterchain_block_handle_->id(), manager_, + td::Timestamp::in(600), std::move(promise)); } void ShardClient::new_masterchain_block_notification(BlockHandle handle, td::Ref state) { diff --git a/validator/shard-client.hpp b/validator/shard-client.hpp index 2025041f..9ad62da2 100644 --- a/validator/shard-client.hpp +++ b/validator/shard-client.hpp @@ -36,6 +36,7 @@ class ShardClient : public td::actor::Actor { bool waiting_ = false; bool init_mode_ = false; + bool started_ = false; td::actor::ActorId manager_; @@ -67,6 +68,7 @@ class ShardClient : public td::actor::Actor { void start_up() override; void start_up_init_mode(); void start_up_init_mode_finished(); + void start(); void got_state_from_db(BlockIdExt masterchain_block_id); void im_download_shard_state(BlockIdExt block_id, td::Promise promise); diff --git a/validator/validate-broadcast.cpp b/validator/validate-broadcast.cpp index c20539b0..2d0fbdab 100644 --- a/validator/validate-broadcast.cpp +++ b/validator/validate-broadcast.cpp @@ -110,12 +110,12 @@ void ValidateBroadcast::start_up() { } else if (key_block_seqno == last_masterchain_state_->get_seqno()) { got_key_block_handle(last_masterchain_block_handle_); } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &ValidateBroadcast::abort_query, R.move_as_error_prefix("cannot find reference key block id: ")); } else { - td::actor::send_closure(SelfId, &ValidateBroadcast::got_key_block_id, R.move_as_ok()); + td::actor::send_closure(SelfId, &ValidateBroadcast::got_key_block_handle, R.move_as_ok()); } }); td::actor::send_closure(manager_, &ValidatorManager::get_block_by_seqno_from_db, @@ -305,7 +305,9 @@ void ValidateBroadcast::checked_proof() { } }); - td::actor::create_actor("applyblock", handle_->id(), data_, manager_, timeout_, std::move(P)).release(); + td::actor::create_actor("applyblock", handle_->id(), data_, handle_->id(), manager_, timeout_, + std::move(P)) + .release(); } else { finish_query(); } diff --git a/validator/validator.h b/validator/validator.h index 1387fc95..5f6a377f 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -117,6 +117,8 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise promise) = 0; virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) = 0; + virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, + td::Promise promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; }; @@ -157,6 +159,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_proof(BlockHandle handle, td::Promise promise) = 0; virtual void get_block_proof_link(BlockHandle handle, td::Promise promise) = 0; virtual void get_block_handle(BlockIdExt block_id, bool force, td::Promise promise) = 0; + virtual void get_key_block_proof(BlockIdExt block_id, td::Promise promise) = 0; + virtual void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_next_key_blocks(BlockIdExt block_id, td::uint32 cnt, td::Promise> promise) = 0; virtual void get_next_block(BlockIdExt block_id, td::Promise promise) = 0; @@ -184,11 +188,15 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_proof_link_from_db_short(BlockIdExt id, td::Promise> promise) = 0; virtual void get_block_by_lt_from_db(AccountIdPrefixFull account, LogicalTime lt, - td::Promise promise) = 0; + td::Promise promise) = 0; virtual void get_block_by_unix_time_from_db(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) = 0; + td::Promise promise) = 0; virtual void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) = 0; + td::Promise promise) = 0; + + virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; + virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, + td::Promise promise) = 0; virtual void run_ext_query(td::BufferSlice data, td::Promise promise) = 0; virtual void prepare_stats(td::Promise>> promise) = 0;