diff --git a/Changelog.md b/Changelog.md index be18efc2..9fd649c4 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,18 +1,28 @@ -## 03.2023 Update +## 2023.04 Update +1. CPU load optimization: previous DHT reconnect policy was too aggressive +2. Network throughput improvements: granular control on external message broadcast, optimize celldb GC, adjust state serialization and block downloading timings, rldp2 for states and archives +3. Update for Fift (namespaces) and Fift libraries (list of improvements: https://github.com/ton-blockchain/ton/issues/631) +4. Better handling of incorrect inputs in funC: fix UB and prevent crashes on some inputs, improve optimizing int consts and unused variables in FunC, fix analyzing repeat loop. FunC version is increase to 0.4.3. +5. `listBlockTransactionsExt` in liteserver added +6. Tvm emulator improvements + +Besides the work of the core team, this update is based on the efforts of @krigga (tvm emulator improvement), @ex3ndr (`PUSHSLICE` fift-asm improvement) and [sec3-service](https://github.com/sec3-service) security auditors (funC improvements). + +## 2023.03 Update 1. Improvement of ADNL connection stability 2. Transaction emulator support and getAccountStateByTransaction method 3. Fixes of typos, undefined behavior and timer warnings 4. Handling incorrect integer literal values in funC; funC version bumped to 0.4.2 5. FunC Mathlib -## 01.2023 Update +## 2023.01 Update 1. Added ConfigParam 44: `SuspendedAddressList`. Upon being set this config suspends initialisation of **uninit** addresses from the list for given time. 2. FunC: `v0.4.1` added pragmas for precise control of computation order 3. FunC: fixed compiler crashes for some exotic inputs 4. FunC: added legacy tester, a collection of smart-contracts which is used to check whether compilator update change compilation result 5. Improved archive manager: proper handling of recently garbage-collected blocks -## 12.2022 Update +## 2022.12 Update Node update: 1. Improvements of ton-proxy: fixed few bugs, improved stability 2. Improved collator/validator checks, added optimization of storage stat calculation, generation and validation of new blocks is made safer @@ -27,7 +37,7 @@ Node update: Besides the work of the core team, this update is based on the efforts of @vtamara (help with abseil-cpp upgrade), @krigga(in-place modification of global variables) and third-party security auditors. -## 10.2022 Update +## 2022.10 Update * Added extended block creation and general perfomance stats gathering * Forbidden report data on blocks not committed to the master chain for LS * Improved debug in TVM @@ -39,7 +49,7 @@ Besides the work of the core team, this update is based on the efforts of @vtama Besides the work of the core team, this update is based on the efforts of @tvorogme (debug improvements), @AlexeyFSL (WASM builds) and third-party security auditors. -## 08.2022 Update +## 2022.08 Update * Blockchain state serialization now works via separate db-handler which simplfies memory clearing after serialization * CellDB now works asynchronously which substantially increase database access throughput * Abseil-cpp and crc32 updated: solve issues with compilation on recent OS distributives @@ -54,7 +64,7 @@ Update coincided with persistent state serialization event which lead to block p Besides the work of the core team, this update is based on the efforts of @awesome-doge (help with abseil-cpp upgrade), @rec00rsiff (noted issues for exotic endianess and implemented network stats) and third-party security auditors. -## 05.2022 Update +## 2022.05 Update * Initial synchronization improved: adjusted timeouts for state download and the way of choosing which state to download. Nodes with low network speed and/or bad connectivity will synchronize faster and consistently. * Improved peer-to-peer network stability and DDoS resistance: now peers will only relay valid messages to the network. Large messages, which require splitting for relaying, will be retranslated as well, but only after the node gets all parts, and reassembles and checks them. Validators may sign certificates for network peers, which allow relaying large messages by parts without checks. It is used now by validators to faster relay new blocks. Sign and import certificate commands are exposed via `validator-engine-console`. * Fixed some rare edge cases in TVM arithmetic operations related to big numbers (`2**63+`) diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 9e106585..44979885 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -944,7 +944,6 @@ void AdnlPeerPairImpl::got_data_from_dht(td::Result R) { CHECK(dht_query_active_); dht_query_active_ = false; next_dht_query_at_ = td::Timestamp::in(td::Random::fast(60.0, 120.0)); - alarm_timestamp().relax(next_dht_query_at_); if (R.is_error()) { VLOG(ADNL_INFO) << this << ": dht query failed: " << R.move_as_error(); return; diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index a1b05b81..598169f7 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -143,6 +143,7 @@ set(FIFT_SOURCE fift/Dictionary.cpp fift/Fift.cpp fift/IntCtx.cpp + fift/HashMap.cpp fift/Continuation.cpp fift/SourceLookup.cpp fift/utils.cpp @@ -151,6 +152,7 @@ set(FIFT_SOURCE fift/Dictionary.h fift/Fift.h fift/IntCtx.h + fift/HashMap.h fift/Continuation.h fift/SourceLookup.h fift/utils.h @@ -300,6 +302,10 @@ target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto) add_library(fift-lib ${FIFT_SOURCE}) target_include_directories(fift-lib PUBLIC $) target_link_libraries(fift-lib PUBLIC ton_crypto ton_db tdutils ton_block) +if (USE_EMSCRIPTEN) + target_link_options(fift-lib PRIVATE -fexceptions) + target_compile_options(fift-lib PRIVATE -fexceptions) +endif() set_target_properties(fift-lib PROPERTIES OUTPUT_NAME fift) add_executable(fift fift/fift-main.cpp) @@ -328,17 +334,20 @@ if (USE_EMSCRIPTEN) add_executable(funcfiftlib funcfiftlib/funcfiftlib.cpp ${FUNC_LIB_SOURCE}) target_include_directories(funcfiftlib PUBLIC $) target_link_libraries(funcfiftlib PUBLIC fift-lib src_parser git) - target_link_options(funcfiftlib PRIVATE -sEXPORTED_RUNTIME_METHODS=FS,ccall,cwrap,_malloc,free,UTF8ToString,stringToUTF8) - target_link_options(funcfiftlib PRIVATE -sEXPORTED_FUNCTIONS=_func_compile,_version) + target_link_options(funcfiftlib PRIVATE -sEXPORTED_RUNTIME_METHODS=FS,ccall,cwrap,UTF8ToString,stringToUTF8,lengthBytesUTF8,addFunction,removeFunction,setValue) + target_link_options(funcfiftlib PRIVATE -sEXPORTED_FUNCTIONS=_func_compile,_version,_malloc,_free,_setThrew) target_link_options(funcfiftlib PRIVATE -sEXPORT_NAME=CompilerModule) target_link_options(funcfiftlib PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0) - target_link_options(funcfiftlib PRIVATE -sFILESYSTEM=1) + target_link_options(funcfiftlib PRIVATE -sFILESYSTEM=1 -lnodefs.js) target_link_options(funcfiftlib PRIVATE -Oz) target_link_options(funcfiftlib PRIVATE -sIGNORE_MISSING_MAIN=1) target_link_options(funcfiftlib PRIVATE -sAUTO_NATIVE_LIBRARIES=0) target_link_options(funcfiftlib PRIVATE -sMODULARIZE=1) + target_link_options(funcfiftlib PRIVATE -sALLOW_MEMORY_GROWTH=1) + target_link_options(funcfiftlib PRIVATE -sALLOW_TABLE_GROWTH=1) target_link_options(funcfiftlib PRIVATE --embed-file ${CMAKE_CURRENT_SOURCE_DIR}/fift/lib@/fiftlib) - target_compile_options(funcfiftlib PRIVATE -sDISABLE_EXCEPTION_CATCHING=0) + target_link_options(funcfiftlib PRIVATE -fexceptions) + target_compile_options(funcfiftlib PRIVATE -fexceptions) endif() add_executable(tlbc tl/tlbc.cpp) diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp index 4abba64c..431a03fe 100644 --- a/crypto/block/check-proof.cpp +++ b/crypto/block/check-proof.cpp @@ -315,6 +315,113 @@ td::Result TransactionList::validate() const { return std::move(res); } +td::Result BlockTransaction::validate(bool check_proof) const { + if (root.is_null()) { + return td::Status::Error("transactions are expected to be non-empty"); + } + if (check_proof && proof->get_hash().bits().compare(root->get_hash().bits(), 256)) { + return td::Status::Error(PSLICE() << "transaction hash mismatch: Merkle proof expects " + << proof->get_hash().bits().to_hex(256) + << " but received data has " << root->get_hash().bits().to_hex(256)); + } + block::gen::Transaction::Record trans; + if (!tlb::unpack_cell(root, trans)) { + return td::Status::Error("cannot unpack transaction cell"); + } + Info res; + res.blkid = blkid; + res.now = trans.now; + res.lt = trans.lt; + res.hash = root->get_hash().bits(); + res.transaction = root; + return std::move(res); +} + +td::Result BlockTransactionList::validate(bool check_proof) const { + constexpr int max_answer_transactions = 256; + + TRY_RESULT_PREFIX(list, vm::std_boc_deserialize_multi(std::move(transactions_boc)), "cannot deserialize transactions boc: "); + std::vector> tx_proofs(list.size()); + + if (check_proof) { + try { + TRY_RESULT(proof_cell, vm::std_boc_deserialize(std::move(proof_boc))); + auto virt_root = vm::MerkleProof::virtualize(proof_cell, 1); + + if (blkid.root_hash != virt_root->get_hash().bits()) { + return td::Status::Error("Invalid block proof root hash"); + } + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(std::move(blk.extra), extra))) { + return td::Status::Error("Error unpacking proof cell"); + } + vm::AugmentedDictionary acc_dict{vm::load_cell_slice_ref(extra.account_blocks), 256, + block::tlb::aug_ShardAccountBlocks}; + + bool eof = false; + ton::LogicalTime reverse = reverse_mode ? ~0ULL : 0; + ton::LogicalTime trans_lt = static_cast(start_lt); + td::Bits256 cur_addr = start_addr; + bool allow_same = true; + int count = 0; + while (!eof && count < req_count && count < max_answer_transactions) { + auto value = acc_dict.extract_value( + acc_dict.vm::DictionaryFixed::lookup_nearest_key(cur_addr.bits(), 256, !reverse, allow_same)); + if (value.is_null()) { + eof = true; + break; + } + allow_same = false; + if (cur_addr != start_addr) { + trans_lt = reverse; + } + + block::gen::AccountBlock::Record acc_blk; + if (!tlb::csr_unpack(std::move(value), acc_blk) || acc_blk.account_addr != cur_addr) { + return td::Status::Error("Error unpacking proof account block"); + } + vm::AugmentedDictionary trans_dict{vm::DictNonEmpty(), std::move(acc_blk.transactions), 64, + block::tlb::aug_AccountTransactions}; + td::BitArray<64> cur_trans{(long long)trans_lt}; + while (count < req_count && count < max_answer_transactions) { + auto tvalue = trans_dict.extract_value_ref( + trans_dict.vm::DictionaryFixed::lookup_nearest_key(cur_trans.bits(), 64, !reverse)); + if (tvalue.is_null()) { + trans_lt = reverse; + break; + } + if (static_cast(count) < tx_proofs.size()) { + tx_proofs[count] = std::move(tvalue); + } + count++; + } + } + if (static_cast(count) != list.size()) { + return td::Status::Error(PSLICE() << "Txs count mismatch in proof (" << count << ") and response (" << list.size() << ")"); + } + } catch (vm::VmError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (vm::VmVirtError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (...) { + return td::Status::Error("Unknown exception raised while verifying proof"); + } + } + + Info res; + for (int i = 0; i < static_cast(list.size()); i++) { + auto& root = list[i]; + BlockTransaction transaction; + transaction.root = root; + transaction.blkid = blkid; + transaction.proof = tx_proofs[i]; + TRY_RESULT(info, transaction.validate(check_proof)); + res.transactions.push_back(std::move(info)); + } + return std::move(res); +} + td::Status BlockProofLink::validate(td::uint32* save_utime) const { if (save_utime) { *save_utime = 0; diff --git a/crypto/block/check-proof.h b/crypto/block/check-proof.h index 527f3138..497a4eba 100644 --- a/crypto/block/check-proof.h +++ b/crypto/block/check-proof.h @@ -88,4 +88,36 @@ struct TransactionList { td::Result validate() const; }; +struct BlockTransaction { + ton::BlockIdExt blkid; + td::Ref root; + td::Ref proof; + + struct Info { + ton::BlockIdExt blkid; + td::uint32 now; + ton::LogicalTime lt; + ton::Bits256 hash; + td::Ref transaction; + }; + td::Result validate(bool check_proof) const; +}; + +struct BlockTransactionList { + ton::BlockIdExt blkid; + td::BufferSlice transactions_boc; + td::BufferSlice proof_boc; + ton::LogicalTime start_lt; + td::Bits256 start_addr; + bool reverse_mode; + int req_count; + + struct Info { + ton::BlockIdExt blkid; + std::vector transactions; + }; + + td::Result validate(bool check_proof) const; +}; + } // namespace block diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index a7500713..183da0a7 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -47,6 +47,7 @@ #include "fift/Fift.h" #include "fift/Dictionary.h" #include "fift/SourceLookup.h" +#include "fift/IntCtx.h" #include "fift/words.h" #include "td/utils/logging.h" @@ -866,8 +867,9 @@ int main(int argc, char* const argv[]) { case 'v': new_verbosity_level = VERBOSITY_NAME(FATAL) + (verbosity = td::to_integer(td::Slice(optarg))); break; - case 'V': - std::cout << "create-state build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + case 'V': + std::cout << "create-state build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); break; case 'h': diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 47ea4e47..fcabf0c0 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -704,7 +704,9 @@ bool Transaction::prepare_storage_phase(const StoragePhaseConfig& cfg, bool forc switch (acc_status) { case Account::acc_uninit: case Account::acc_frozen: - if (total_due > cfg.delete_due_limit) { + if (total_due > cfg.delete_due_limit && balance.extra.is_null()) { + // Keeping accounts with non-null extras is a temporary measure before implementing proper collection of + // extracurrencies from deleted accounts res->deleted = true; acc_status = Account::acc_deleted; if (balance.extra.not_null()) { diff --git a/crypto/fift/Continuation.cpp b/crypto/fift/Continuation.cpp index 7e3b5ea2..e895082f 100644 --- a/crypto/fift/Continuation.cpp +++ b/crypto/fift/Continuation.cpp @@ -27,7 +27,7 @@ namespace fift { // bool FiftCont::print_dict_name(std::ostream& os, const IntCtx& ctx) const { std::string word_name; - if (ctx.dictionary && ctx.dictionary->lookup_def(this, &word_name)) { + if (ctx.dictionary.lookup_def(this, &word_name)) { if (word_name.size() && word_name.back() == ' ') { word_name.pop_back(); } @@ -39,7 +39,7 @@ bool FiftCont::print_dict_name(std::ostream& os, const IntCtx& ctx) const { std::string FiftCont::get_dict_name(const IntCtx& ctx) const { std::string word_name; - if (ctx.dictionary && ctx.dictionary->lookup_def(this, &word_name)) { + if (ctx.dictionary.lookup_def(this, &word_name)) { if (word_name.size() && word_name.back() == ' ') { word_name.pop_back(); } @@ -63,6 +63,140 @@ bool FiftCont::dump(std::ostream& os, const IntCtx& ctx) const { return ok; } +// +// StackWord +// +Ref StackWord::run_tail(IntCtx& ctx) const { + f(ctx.stack); + return {}; +} + +// +// CtxWord +// +Ref CtxWord::run_tail(IntCtx& ctx) const { + f(ctx); + return {}; +} + +// +// CtxTailWord +// +Ref CtxTailWord::run_tail(IntCtx& ctx) const { + return f(ctx); +} + +// +// WordList +// +WordList::WordList(std::vector>&& _list) : list(std::move(_list)) { +} + +WordList::WordList(const std::vector>& _list) : list(_list) { +} + +WordList& WordList::push_back(Ref word_def) { + list.push_back(std::move(word_def)); + return *this; +} + +WordList& WordList::push_back(FiftCont& wd) { + list.emplace_back(&wd); + return *this; +} + +Ref WordList::run_tail(IntCtx& ctx) const { + if (list.empty()) { + return {}; + } + if (list.size() > 1) { + ctx.next = td::make_ref(std::move(ctx.next), Ref(this), 1); + } + return list[0]; +} + +void WordList::close() { + list.shrink_to_fit(); +} + +WordList& WordList::append(const std::vector>& other) { + list.insert(list.end(), other.begin(), other.end()); + return *this; +} + +WordList& WordList::append(const Ref* begin, const Ref* end) { + list.insert(list.end(), begin, end); + return *this; +} + +bool WordList::dump(std::ostream& os, const IntCtx& ctx) const { + os << "{"; + for (auto entry : list) { + os << ' '; + entry->print_name(os, ctx); + } + os << " }" << std::endl; + return true; +} + +// +// ListCont +// + +Ref ListCont::run_tail(IntCtx& ctx) const { + auto sz = list->size(); + if (pos >= sz) { + return std::move(ctx.next); + } else if (ctx.next.not_null()) { + ctx.next = td::make_ref(SeqCont::seq(next, std::move(ctx.next)), list, pos + 1); + } else if (pos + 1 == sz) { + ctx.next = next; + } else { + ctx.next = td::make_ref(next, list, pos + 1); + } + return list->at(pos); +} + +Ref ListCont::run_modify(IntCtx& ctx) { + auto sz = list->size(); + if (pos >= sz) { + return std::move(ctx.next); + } + auto cur = list->at(pos++); + if (ctx.next.not_null()) { + next = SeqCont::seq(next, std::move(ctx.next)); + } + if (pos == sz) { + ctx.next = std::move(next); + } else { + ctx.next = self(); + } + return cur; +} + +bool ListCont::dump(std::ostream& os, const IntCtx& ctx) const { + std::string dict_name = list->get_dict_name(ctx); + if (!dict_name.empty()) { + os << "[in " << dict_name << ":] "; + } + std::size_t sz = list->size(), i, a = (pos >= 16 ? pos - 16 : 0), b = std::min(pos + 16, sz); + if (a > 0) { + os << "... "; + } + for (i = a; i < b; i++) { + if (i == pos) { + os << "**HERE** "; + } + list->at(i)->print_name(os, ctx); + os << ' '; + } + if (b < sz) { + os << "..."; + } + os << std::endl; + return true; +} + // // QuitCont // @@ -295,12 +429,15 @@ bool GenericLitCont::print_name(std::ostream& os, const IntCtx& ctx) const { bool sp = false; for (auto entry : list) { if (sp) { - os << sp; + os << ' '; } sp = true; int tp = entry.type(); if (entry.is_int() || entry.is(vm::StackEntry::t_string) || entry.is(vm::StackEntry::t_bytes)) { entry.dump(os); + } else if (entry.is_atom()) { + os << '`'; + entry.dump(os); } else { auto cont_lit = entry.as_object(); if (cont_lit.not_null()) { diff --git a/crypto/fift/Continuation.h b/crypto/fift/Continuation.h index f2c44e7b..6623b642 100644 --- a/crypto/fift/Continuation.h +++ b/crypto/fift/Continuation.h @@ -17,6 +17,7 @@ Copyright 2020 Telegram Systems LLP */ #pragma once +#include #include "common/refcnt.hpp" #include "common/refint.h" #include "vm/stack.hpp" @@ -76,6 +77,101 @@ class FiftCont : public td::CntObject { } }; +typedef std::function StackWordFunc; +typedef std::function CtxWordFunc; +typedef std::function(IntCtx&)> CtxTailWordFunc; + +class NopWord : public FiftCont { + public: + NopWord() = default; + ~NopWord() override = default; + Ref run_tail(IntCtx& ctx) const override { + return {}; + } +}; + +class StackWord : public FiftCont { + StackWordFunc f; + + public: + StackWord(StackWordFunc _f) : f(std::move(_f)) { + } + ~StackWord() override = default; + Ref run_tail(IntCtx& ctx) const override; +}; + +class CtxWord : public FiftCont { + CtxWordFunc f; + + public: + CtxWord(CtxWordFunc _f) : f(std::move(_f)) { + } + ~CtxWord() override = default; + Ref run_tail(IntCtx& ctx) const override; +}; + +class CtxTailWord : public FiftCont { + CtxTailWordFunc f; + + public: + CtxTailWord(CtxTailWordFunc _f) : f(std::move(_f)) { + } + ~CtxTailWord() override = default; + Ref run_tail(IntCtx& ctx) const override; +}; + +class WordList : public FiftCont { + std::vector> list; + + public: + ~WordList() override = default; + WordList() = default; + WordList(std::vector>&& _list); + WordList(const std::vector>& _list); + WordList& push_back(Ref word_def); + WordList& push_back(FiftCont& wd); + Ref run_tail(IntCtx& ctx) const override; + void close(); + bool is_list() const override { + return true; + } + long long list_size() const override { + return (long long)list.size(); + } + std::size_t size() const { + return list.size(); + } + const Ref& at(std::size_t idx) const { + return list.at(idx); + } + const Ref* get_list() const override { + return list.data(); + } + WordList& append(const std::vector>& other); + WordList& append(const Ref* begin, const Ref* end); + WordList* make_copy() const override { + return new WordList(list); + } + bool dump(std::ostream& os, const IntCtx& ctx) const override; +}; + +class ListCont : public FiftCont { + Ref next; + Ref list; + std::size_t pos; + + public: + ListCont(Ref nxt, Ref wl, std::size_t p = 0) : next(std::move(nxt)), list(std::move(wl)), pos(p) { + } + ~ListCont() override = default; + Ref run_tail(IntCtx& ctx) const override; + Ref run_modify(IntCtx& ctx) override; + Ref up() const override { + return next; + } + bool dump(std::ostream& os, const IntCtx& ctx) const override; +}; + class QuitCont : public FiftCont { int exit_code; diff --git a/crypto/fift/Dictionary.cpp b/crypto/fift/Dictionary.cpp index 59da278f..d2eae0a3 100644 --- a/crypto/fift/Dictionary.cpp +++ b/crypto/fift/Dictionary.cpp @@ -17,143 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "Dictionary.h" +#include "IntCtx.h" namespace fift { -// -// StackWord -// -Ref StackWord::run_tail(IntCtx& ctx) const { - f(ctx.stack); - return {}; -} - -// -// CtxWord -// -Ref CtxWord::run_tail(IntCtx& ctx) const { - f(ctx); - return {}; -} - -// -// CtxTailWord -// -Ref CtxTailWord::run_tail(IntCtx& ctx) const { - return f(ctx); -} - -// -// WordList -// -WordList::WordList(std::vector>&& _list) : list(std::move(_list)) { -} - -WordList::WordList(const std::vector>& _list) : list(_list) { -} - -WordList& WordList::push_back(Ref word_def) { - list.push_back(std::move(word_def)); - return *this; -} - -WordList& WordList::push_back(FiftCont& wd) { - list.emplace_back(&wd); - return *this; -} - -Ref WordList::run_tail(IntCtx& ctx) const { - if (list.empty()) { - return {}; - } - if (list.size() > 1) { - ctx.next = td::make_ref(std::move(ctx.next), Ref(this), 1); - } - return list[0]; -} - -void WordList::close() { - list.shrink_to_fit(); -} - -WordList& WordList::append(const std::vector>& other) { - list.insert(list.end(), other.begin(), other.end()); - return *this; -} - -WordList& WordList::append(const Ref* begin, const Ref* end) { - list.insert(list.end(), begin, end); - return *this; -} - -bool WordList::dump(std::ostream& os, const IntCtx& ctx) const { - os << "{"; - for (auto entry : list) { - os << ' '; - entry->print_name(os, ctx); - } - os << " }" << std::endl; - return true; -} - -// -// ListCont -// - -Ref ListCont::run_tail(IntCtx& ctx) const { - auto sz = list->size(); - if (pos >= sz) { - return std::move(ctx.next); - } else if (ctx.next.not_null()) { - ctx.next = td::make_ref(SeqCont::seq(next, std::move(ctx.next)), list, pos + 1); - } else if (pos + 1 == sz) { - ctx.next = next; - } else { - ctx.next = td::make_ref(next, list, pos + 1); - } - return list->at(pos); -} - -Ref ListCont::run_modify(IntCtx& ctx) { - auto sz = list->size(); - if (pos >= sz) { - return std::move(ctx.next); - } - auto cur = list->at(pos++); - if (ctx.next.not_null()) { - next = SeqCont::seq(next, std::move(ctx.next)); - } - if (pos == sz) { - ctx.next = std::move(next); - } else { - ctx.next = self(); - } - return cur; -} - -bool ListCont::dump(std::ostream& os, const IntCtx& ctx) const { - std::string dict_name = list->get_dict_name(ctx); - if (!dict_name.empty()) { - os << "[in " << dict_name << ":] "; - } - std::size_t sz = list->size(), i, a = (pos >= 16 ? pos - 16 : 0), b = std::min(pos + 16, sz); - if (a > 0) { - os << "... "; - } - for (i = a; i < b; i++) { - if (i == pos) { - os << "**HERE** "; - } - list->at(i)->print_name(os, ctx); - os << ' '; - } - if (b < sz) { - os << "..."; - } - os << std::endl; - return true; -} - // // DictEntry // @@ -167,15 +34,49 @@ DictEntry::DictEntry(CtxWordFunc func, bool _act) : def(Ref{true, std:: DictEntry::DictEntry(CtxTailWordFunc func, bool _act) : def(Ref{true, std::move(func)}), active(_act) { } +DictEntry DictEntry::create_from(vm::StackEntry se) { + if (se.is_tuple()) { + auto& tuple = *se.as_tuple(); + if (tuple.size() == 1) { + auto def = tuple[0].as_object(); + if (def.not_null()) { + return DictEntry{std::move(def), true}; + } + } + } else { + auto def = std::move(se).as_object(); + if (def.not_null()) { + return DictEntry{std::move(def)}; + } + } + return {}; +} + +DictEntry::operator vm::StackEntry() const& { + if (def.is_null()) { + return {}; + } else if (active) { + return vm::make_tuple_ref(vm::StackEntry{vm::from_object, def}); + } else { + return {vm::from_object, def}; + } +} + +DictEntry::operator vm::StackEntry() && { + if (def.is_null()) { + return {}; + } else if (active) { + return vm::make_tuple_ref(vm::StackEntry{vm::from_object, std::move(def)}); + } else { + return {vm::from_object, std::move(def)}; + } +} + // // Dictionary // -DictEntry* Dictionary::lookup(td::Slice name) { - auto it = words_.find(name); - if (it == words_.end()) { - return nullptr; - } - return &it->second; +DictEntry Dictionary::lookup(std::string name) const { + return DictEntry::create_from(words().get(name)); } void Dictionary::def_ctx_word(std::string name, CtxWordFunc func) { @@ -196,26 +97,27 @@ void Dictionary::def_ctx_tail_word(std::string name, CtxTailWordFunc func) { } void Dictionary::def_word(std::string name, DictEntry word) { - auto res = words_.emplace(name, std::move(word)); - LOG_IF(FATAL, !res.second) << "Cannot redefine word: " << name; + auto dict = words(); + dict.set(std::move(name), vm::StackEntry(std::move(word))); + set_words(dict); } -void Dictionary::undef_word(td::Slice name) { - auto it = words_.find(name); - if (it == words_.end()) { - return; +void Dictionary::undef_word(std::string name) { + auto dict = words(); + if (dict.remove(name)) { + set_words(dict); } - words_.erase(it); } bool Dictionary::lookup_def(const FiftCont* cont, std::string* word_ptr) const { if (!cont) { return false; } - for (const auto& entry : words_) { - if (entry.second.get_def().get() == cont) { + for (auto entry : words()) { + auto val = DictEntry::create_from(entry.value()); + if (val.get_def().get() == cont && entry.key().is_string()) { if (word_ptr) { - *word_ptr = entry.first; + *word_ptr = vm::StackEntry(entry.key()).as_string(); } return true; } @@ -223,35 +125,4 @@ bool Dictionary::lookup_def(const FiftCont* cont, std::string* word_ptr) const { return false; } -void interpret_nop(vm::Stack& stack) { -} - -Ref Dictionary::nop_word_def = Ref{true, interpret_nop}; - -// -// functions for wordef -// -Ref pop_exec_token(vm::Stack& stack) { - stack.check_underflow(1); - auto wd_ref = stack.pop().as_object(); - if (wd_ref.is_null()) { - throw IntError{"execution token expected"}; - } - return wd_ref; -} - -Ref pop_word_list(vm::Stack& stack) { - stack.check_underflow(1); - auto wl_ref = stack.pop().as_object(); - if (wl_ref.is_null()) { - throw IntError{"word list expected"}; - } - return wl_ref; -} - -void push_argcount(vm::Stack& stack, int args) { - stack.push_smallint(args); - stack.push({vm::from_object, Dictionary::nop_word_def}); -} - } // namespace fift diff --git a/crypto/fift/Dictionary.h b/crypto/fift/Dictionary.h index 7307cdbe..b24bc742 100644 --- a/crypto/fift/Dictionary.h +++ b/crypto/fift/Dictionary.h @@ -17,115 +17,27 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once - -#include -#include - -#include "IntCtx.h" #include "Continuation.h" +#include "HashMap.h" +#include "vm/box.hpp" namespace fift { using td::Ref; +struct IntCtx; + /* * * WORD CLASSES * */ -typedef std::function StackWordFunc; -typedef std::function CtxWordFunc; - -class StackWord : public FiftCont { - StackWordFunc f; - - public: - StackWord(StackWordFunc _f) : f(std::move(_f)) { - } - ~StackWord() override = default; - Ref run_tail(IntCtx& ctx) const override; -}; - -class CtxWord : public FiftCont { - CtxWordFunc f; - - public: - CtxWord(CtxWordFunc _f) : f(std::move(_f)) { - } - ~CtxWord() override = default; - Ref run_tail(IntCtx& ctx) const override; -}; - -typedef std::function(IntCtx&)> CtxTailWordFunc; - -class CtxTailWord : public FiftCont { - CtxTailWordFunc f; - - public: - CtxTailWord(CtxTailWordFunc _f) : f(std::move(_f)) { - } - ~CtxTailWord() override = default; - Ref run_tail(IntCtx& ctx) const override; -}; - -class WordList : public FiftCont { - std::vector> list; - - public: - ~WordList() override = default; - WordList() = default; - WordList(std::vector>&& _list); - WordList(const std::vector>& _list); - WordList& push_back(Ref word_def); - WordList& push_back(FiftCont& wd); - Ref run_tail(IntCtx& ctx) const override; - void close(); - bool is_list() const override { - return true; - } - long long list_size() const override { - return (long long)list.size(); - } - std::size_t size() const { - return list.size(); - } - const Ref& at(std::size_t idx) const { - return list.at(idx); - } - const Ref* get_list() const override { - return list.data(); - } - WordList& append(const std::vector>& other); - WordList& append(const Ref* begin, const Ref* end); - WordList* make_copy() const override { - return new WordList(list); - } - bool dump(std::ostream& os, const IntCtx& ctx) const override; -}; - -class ListCont : public FiftCont { - Ref next; - Ref list; - std::size_t pos; - - public: - ListCont(Ref nxt, Ref wl, std::size_t p = 0) : next(std::move(nxt)), list(std::move(wl)), pos(p) { - } - ~ListCont() override = default; - Ref run_tail(IntCtx& ctx) const override; - Ref run_modify(IntCtx& ctx) override; - Ref up() const override { - return next; - } - bool dump(std::ostream& os, const IntCtx& ctx) const override; -}; - class DictEntry { Ref def; - bool active; + bool active{false}; public: - DictEntry() = delete; + DictEntry() = default; DictEntry(const DictEntry& ref) = default; DictEntry(DictEntry&& ref) = default; DictEntry(Ref _def, bool _act = false) : def(std::move(_def)), active(_act) { @@ -137,6 +49,9 @@ class DictEntry { //DictEntry(std::vector>&& word_list); DictEntry& operator=(const DictEntry&) = default; DictEntry& operator=(DictEntry&&) = default; + static DictEntry create_from(vm::StackEntry se); + explicit operator vm::StackEntry() const&; + explicit operator vm::StackEntry() &&; Ref get_def() const& { return def; } @@ -146,16 +61,17 @@ class DictEntry { bool is_active() const { return active; } + bool empty() const { + return def.is_null(); + } + explicit operator bool() const { + return def.not_null(); + } + bool operator!() const { + return def.is_null(); + } }; -/* -DictEntry::DictEntry(const std::vector>& word_list) : def(Ref{true, word_list}) { -} - -DictEntry::DictEntry(std::vector>&& word_list) : def(Ref{true, std::move(word_list)}) { -} -*/ - /* * * DICTIONARIES @@ -164,37 +80,52 @@ DictEntry::DictEntry(std::vector>&& word_list) : def(Ref class Dictionary { public: - DictEntry* lookup(td::Slice name); + Dictionary() : box_(true) { + } + Dictionary(Ref box) : box_(std::move(box)) { + } + Dictionary(Ref hmap) : box_(true, vm::from_object, std::move(hmap)) { + } + + DictEntry lookup(std::string name) const; void def_ctx_word(std::string name, CtxWordFunc func); void def_ctx_tail_word(std::string name, CtxTailWordFunc func); void def_active_word(std::string name, CtxWordFunc func); void def_stack_word(std::string name, StackWordFunc func); void def_word(std::string name, DictEntry word); - void undef_word(td::Slice name); + void undef_word(std::string name); bool lookup_def(const FiftCont* cont, std::string* word_ptr = nullptr) const; bool lookup_def(Ref cont, std::string* word_ptr = nullptr) const { return lookup_def(cont.get(), word_ptr); } auto begin() const { - return words_.begin(); + return words().begin(); } auto end() const { - return words_.end(); + return words().end(); + } + HashmapKeeper words() const { + if (box_->empty()) { + return {}; + } else { + return box_->get().as_object(); + } + } + Ref get_box() const { + return box_; + } + void set_words(Ref new_words) { + box_->set(vm::StackEntry{vm::from_object, std::move(new_words)}); + } + bool operator==(const Dictionary& other) const { + return box_ == other.box_; + } + bool operator!=(const Dictionary& other) const { + return box_ != other.box_; } - static Ref nop_word_def; - private: - std::map> words_; + Ref box_; }; -/* - * - * AUX FUNCTIONS FOR WORD DEFS - * - */ - -Ref pop_exec_token(vm::Stack& stack); -Ref pop_word_list(vm::Stack& stack); -void push_argcount(vm::Stack& stack, int args); } // namespace fift diff --git a/crypto/fift/Fift.cpp b/crypto/fift/Fift.cpp index ef265657..85511a38 100644 --- a/crypto/fift/Fift.cpp +++ b/crypto/fift/Fift.cpp @@ -17,7 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "Fift.h" - +#include "IntCtx.h" #include "words.h" #include "td/utils/PathView.h" @@ -51,7 +51,7 @@ td::Result Fift::interpret_istream(std::istream& stream, std::string curren td::Result Fift::do_interpret(IntCtx& ctx, bool is_interactive) { ctx.ton_db = &config_.ton_db; ctx.source_lookup = &config_.source_lookup; - ctx.dictionary = &config_.dictionary; + ctx.dictionary = ctx.main_dictionary = ctx.context = config_.dictionary; ctx.output_stream = config_.output_stream; ctx.error_stream = config_.error_stream; if (!ctx.output_stream) { @@ -71,7 +71,7 @@ td::Result Fift::do_interpret(IntCtx& ctx, bool is_interactive) { ctx.top_ctx(); ctx.clear_error(); ctx.stack.clear(); - ctx.load_next_line(); + ctx.parser->load_next_line(); continue; } } diff --git a/crypto/fift/HashMap.cpp b/crypto/fift/HashMap.cpp new file mode 100644 index 00000000..a4ca0e9b --- /dev/null +++ b/crypto/fift/HashMap.cpp @@ -0,0 +1,371 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "HashMap.h" +#include "td/utils/Random.h" +#include "IntCtx.h" + +namespace fift { +using td::Ref; + +DictKey::DictKey(vm::StackEntry se) { + auto tp = tp_ = se.type(); + switch (tp) { + case Type::t_int: + ref_ = se.as_int(); + break; + case Type::t_atom: + ref_ = se.as_atom(); + break; + case Type::t_string: + ref_ = se.as_string_ref(); + break; + case Type::t_bytes: + ref_ = se.as_bytes_ref(); + break; + case Type::t_null: + break; + default: + throw IntError{"unsupported key type"}; + } + compute_hash(); +} + +DictKey::operator vm::StackEntry() const& { + switch (tp_) { + case Type::t_int: + return value(); + case Type::t_atom: + return value(); + case Type::t_string: + case Type::t_bytes: + return {value>(), tp_ == Type::t_bytes}; + default: + return {}; + } +} + +DictKey::operator vm::StackEntry() && { + switch (tp_) { + case Type::t_int: + return move_value(); + case Type::t_atom: + return move_value(); + case Type::t_string: + case Type::t_bytes: + return {move_value>(), tp_ == Type::t_bytes}; + default: + return {}; + } +} + +std::ostream& operator<<(std::ostream& os, const DictKey& dkey) { + return os << vm::StackEntry(dkey).to_string(); +} + +int DictKey::cmp_internal(const DictKey& other) const { + if (tp_ != other.tp_) { + return tp_ < other.tp_ ? -1 : 1; + } + switch (tp_) { + case Type::t_int: + return td::cmp(value(), other.value()); + case Type::t_atom: { + int u = value()->index(), v = other.value()->index(); + return u == v ? 0 : (u < v ? -1 : 1); + } + case Type::t_string: + case Type::t_bytes: + return value>()->compare(*other.value>()); + default: + return 0; + } +} + +int DictKey::cmp(const DictKey& other) const { + if (hash_ < other.hash_) { + return -1; + } else if (hash_ > other.hash_) { + return 1; + } else { + return cmp_internal(other); + } +} + +DictKey::keyhash_t DictKey::compute_str_hash(DictKey::keyhash_t h, const char* str, std::size_t len) { + const char* end = str + len; + while (str < end) { + h = h * StrHash + (unsigned char)*str++; + } + return h; +} + +DictKey::keyhash_t DictKey::compute_int_hash(td::AnyIntView<> x) { + keyhash_t h = IntHash0; + for (int i = 0; i < x.size(); i++) { + h = h * MixConst3 + x.digits[i]; + } + return h * MixConst4; +} + +DictKey::keyhash_t DictKey::compute_hash() { + switch (tp_) { + case Type::t_int: + return hash_ = compute_int_hash(value()->as_any_int()); + case Type::t_atom: + return hash_ = value()->index() * MixConst1 + MixConst2; + case Type::t_string: + case Type::t_bytes: { + auto ref = value>(); + return hash_ = compute_str_hash(tp_, ref->data(), ref->size()); + } + default: + return hash_ = 0; + } +} + +const Hashmap* Hashmap::lookup_key_aux(const Hashmap* root, const DictKey& key) { + if (key.is_null()) { + return nullptr; + } + while (root) { + int r = key.cmp(root->key_); + if (!r) { + break; + } + root = (r < 0 ? root->left_.get() : root->right_.get()); + } + return root; +} + +Ref Hashmap::lookup_key(Ref root, const DictKey& key) { + return Ref(lookup_key_aux(root.get(), key)); +} + +vm::StackEntry Hashmap::get_key(Ref root, const DictKey& key) { + auto node = lookup_key_aux(root.get(), key); + if (node) { + return node->value_; + } else { + return {}; + } +} + +std::pair, vm::StackEntry> Hashmap::get_remove_key(Ref root, const DictKey& key) { + if (root.is_null() || key.is_null()) { + return {std::move(root), {}}; + } + vm::StackEntry val; + auto res = root->get_remove_internal(key, val); + if (val.is_null()) { + return {std::move(root), {}}; + } else { + return {std::move(res), std::move(val)}; + } +} + +Ref Hashmap::remove_key(Ref root, const DictKey& key) { + if (root.is_null() || key.is_null()) { + return root; + } + vm::StackEntry val; + auto res = root->get_remove_internal(key, val); + if (val.is_null()) { + return root; + } else { + return res; + } +} + +Ref Hashmap::get_remove_internal(const DictKey& key, vm::StackEntry& val) const { + int r = key.cmp(key_); + if (!r) { + val = value_; + return merge(left_, right_); + } else if (r < 0) { + if (left_.is_null()) { + return {}; + } else { + auto res = left_->get_remove_internal(key, val); + if (val.is_null()) { + return res; + } else { + return td::make_ref(key_, value_, std::move(res), right_, y_); + } + } + } else if (right_.is_null()) { + return {}; + } else { + auto res = right_->get_remove_internal(key, val); + if (val.is_null()) { + return res; + } else { + return td::make_ref(key_, value_, left_, std::move(res), y_); + } + } +} + +Ref Hashmap::merge(Ref a, Ref b) { + if (a.is_null()) { + return b; + } else if (b.is_null()) { + return a; + } else if (a->y_ > b->y_) { + auto& aa = a.write(); + aa.right_ = merge(std::move(aa.right_), std::move(b)); + return a; + } else { + auto& bb = b.write(); + bb.left_ = merge(std::move(a), std::move(bb.left_)); + return b; + } +} + +Ref Hashmap::set(Ref root, const DictKey& key, vm::StackEntry value) { + if (!key.is_null() && !replace(root, key, value) && !value.is_null()) { + insert(root, key, value, new_y()); + } + return root; +} + +bool Hashmap::replace(Ref& root, const DictKey& key, vm::StackEntry value) { + if (root.is_null() || key.is_null()) { + return false; + } + if (value.is_null()) { + auto res = root->get_remove_internal(key, value); + if (value.is_null()) { + return false; + } else { + root = std::move(res); + return true; + } + } + bool found = false; + auto res = root->replace_internal(key, std::move(value), found); + if (found) { + root = std::move(res); + } + return found; +} + +Ref Hashmap::replace_internal(const DictKey& key, const vm::StackEntry& value, bool& found) const { + int r = key.cmp(key_); + if (!r) { + found = true; + return td::make_ref(key_, value, left_, right_, y_); + } else if (r < 0) { + if (left_.is_null()) { + found = false; + return {}; + } + auto res = left_->replace_internal(key, value, found); + if (!found) { + return {}; + } + return td::make_ref(key_, value_, std::move(res), right_, y_); + } else { + if (right_.is_null()) { + found = false; + return {}; + } + auto res = right_->replace_internal(key, value, found); + if (!found) { + return {}; + } + return td::make_ref(key_, value_, left_, std::move(res), y_); + } +} + +void Hashmap::insert(Ref& root, const DictKey& key, vm::StackEntry value, long long y) { + if (root.is_null()) { + root = td::make_ref(key, std::move(value), empty(), empty(), y); + return; + } + if (root->y_ <= y) { + auto res = split(std::move(root), key); + root = td::make_ref(key, std::move(value), std::move(res.first), std::move(res.second), y); + return; + } + int r = key.cmp(root->key_); + CHECK(r); + insert(r < 0 ? root.write().left_ : root.write().right_, key, std::move(value), y); +} + +std::pair, Ref> Hashmap::split(Ref root, const DictKey& key, bool cmpv) { + if (root.is_null()) { + return {{}, {}}; + } + int r = key.cmp(root->key_); + if (r < (int)cmpv) { + if (root->left_.is_null()) { + return {{}, std::move(root)}; + } + auto res = split(root->left_, key, cmpv); + return {std::move(res.first), + td::make_ref(root->key_, root->value_, std::move(res.second), root->right_, root->y_)}; + } else { + if (root->right_.is_null()) { + return {std::move(root), {}}; + } + auto res = split(root->right_, key, cmpv); + return {td::make_ref(root->key_, root->value_, root->left_, std::move(res.first), root->y_), + std::move(res.second)}; + } +} + +long long Hashmap::new_y() { + return td::Random::fast_uint64(); +} + +bool HashmapIterator::unwind(Ref root) { + if (root.is_null()) { + return false; + } + while (true) { + auto left = root->lr(down_); + if (left.is_null()) { + cur_ = std::move(root); + return true; + } + stack_.push_back(std::move(root)); + root = std::move(left); + } +} + +bool HashmapIterator::next() { + if (cur_.not_null()) { + cur_ = cur_->rl(down_); + if (cur_.not_null()) { + while (true) { + auto left = cur_->lr(down_); + if (left.is_null()) { + return true; + } + stack_.push_back(std::move(cur_)); + cur_ = std::move(left); + } + } + } + if (stack_.empty()) { + return false; + } + cur_ = std::move(stack_.back()); + stack_.pop_back(); + return true; +} + +} // namespace fift diff --git a/crypto/fift/HashMap.h b/crypto/fift/HashMap.h new file mode 100644 index 00000000..d3d9ffbe --- /dev/null +++ b/crypto/fift/HashMap.h @@ -0,0 +1,306 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "common/refcnt.hpp" +#include "vm/stack.hpp" +#include "vm/atom.h" + +namespace fift { + +using td::Ref; +using td::RefAny; + +class DictKey { + public: + typedef vm::StackEntry::Type Type; + typedef unsigned long long keyhash_t; + + private: + RefAny ref_; + Type tp_ = Type::t_null; + keyhash_t hash_ = 0; + + static constexpr keyhash_t IntHash0 = 0xce6ab89d724409ed, MixConst1 = 0xcd5c126501510979, + MixConst2 = 0xb8f44d7fd6274ad1, MixConst3 = 0xd08726ea2422e405, + MixConst4 = 0x6407d2aeb5039dfb, StrHash = 0x93ff128344add06d; + keyhash_t compute_hash(); + static keyhash_t compute_str_hash(DictKey::keyhash_t h, const char* str, std::size_t len); + static keyhash_t compute_int_hash(td::AnyIntView<> x); + int cmp_internal(const DictKey& other) const; + template + Ref value() const { + return Ref{td::static_cast_ref(), ref_}; + } + template + Ref move_value() { + return Ref{td::static_cast_ref(), std::move(ref_)}; + } + + public: + DictKey() : ref_(), tp_(Type::t_null) { + } + DictKey(const DictKey& other) = default; + DictKey(DictKey&& other) = default; + DictKey& operator=(const DictKey& other) = default; + DictKey& operator=(DictKey&& other) = default; + DictKey(Ref atom_ref) : ref_(std::move(atom_ref)), tp_(Type::t_atom) { + compute_hash(); + } + DictKey(td::RefInt256 int_ref) : ref_(std::move(int_ref)), tp_(Type::t_int) { + compute_hash(); + } + explicit DictKey(vm::StackEntry se); + DictKey(std::string str, bool bytes = false) : ref_(), tp_(bytes ? Type::t_bytes : Type::t_string) { + ref_ = Ref>{true, std::move(str)}; + compute_hash(); + } + Type type() const { + return tp_; + } + void swap(DictKey& other) { + ref_.swap(other.ref_); + std::swap(tp_, other.tp_); + } + + operator vm::StackEntry() const&; + operator vm::StackEntry() &&; + int cmp(const DictKey& other) const; + bool operator==(const DictKey& other) const { + return hash_ == other.hash_ && !cmp_internal(other); + } + bool operator!=(const DictKey& other) const { + return hash_ != other.hash_ || cmp_internal(other); + } + bool operator<(const DictKey& other) const { + return hash_ < other.hash_ || (hash_ == other.hash_ && cmp_internal(other) < 0); + } + bool is_null() const { + return tp_ == Type::t_null; + } + bool is_string() const { + return tp_ == Type::t_string; + } +}; + +std::ostream& operator<<(std::ostream& os, const DictKey& dkey); + +class Hashmap : public td::CntObject { + DictKey key_; + vm::StackEntry value_; + Ref left_; + Ref right_; + long long y_; + + public: + Hashmap(DictKey key, vm::StackEntry value, Ref left, Ref right, long long y) + : key_(std::move(key)), value_(std::move(value)), left_(std::move(left)), right_(std::move(right)), y_(y) { + } + Hashmap(const Hashmap& other) = default; + Hashmap(Hashmap&& other) = default; + virtual ~Hashmap() { + } + Hashmap* make_copy() const override { + return new Hashmap(*this); + } + const DictKey& key() const& { + return key_; + } + DictKey key() && { + return std::move(key_); + } + const vm::StackEntry& value() const& { + return value_; + } + vm::StackEntry value() && { + return std::move(value_); + } + Ref left() const { + return left_; + } + Ref right() const { + return right_; + } + Ref lr(bool branch) const { + return branch ? right_ : left_; + } + Ref rl(bool branch) const { + return branch ? left_ : right_; + } + static Ref lookup_key(Ref root, const DictKey& key); + template + static Ref lookup(Ref root, Args&&... args) { + return lookup_key(std::move(root), DictKey{std::forward(args)...}); + } + static vm::StackEntry get_key(Ref root, const DictKey& key); + template + static vm::StackEntry get(Ref root, Args&&... args) { + return get_key(std::move(root), DictKey{std::forward(args)...}); + } + static Ref remove_key(Ref root, const DictKey& key); + template + static Ref remove(Ref root, Args&&... args) { + return remove_key(std::move(root), DictKey{std::forward(args)...}); + } + static std::pair, vm::StackEntry> get_remove_key(Ref root, const DictKey& key); + template + static std::pair, vm::StackEntry> get_remove(Ref root, Args&&... args) { + return get_remove_key(std::move(root), DictKey{std::forward(args)...}); + } + static Ref set(Ref root, const DictKey& key, vm::StackEntry value); + static bool replace(Ref& root, const DictKey& key, vm::StackEntry value); + static std::pair, Ref> split(Ref root, const DictKey& key, bool eq_left = false); + static Ref empty() { + return {}; + } + + private: + static Ref merge(Ref a, Ref b); + static const Hashmap* lookup_key_aux(const Hashmap* root, const DictKey& key); + Ref get_remove_internal(const DictKey& key, vm::StackEntry& val) const; + Ref replace_internal(const DictKey& key, const vm::StackEntry& value, bool& found) const; + static void insert(Ref& root, const DictKey& key, vm::StackEntry value, long long y); + static long long new_y(); +}; + +struct HashmapIdx { + Ref& root_; + DictKey idx_; + template + HashmapIdx(Ref& root, Args&&... args) : root_(root), idx_(std::forward(args)...) { + } + operator vm::StackEntry() const { + return Hashmap::get(root_, idx_); + } + template + HashmapIdx& operator=(T&& value) { + root_ = Hashmap::set(root_, idx_, vm::StackEntry(std::forward(value))); + return *this; + } +}; + +class HashmapIterator { + std::vector> stack_; + Ref cur_; + const bool down_{false}; + bool unwind(Ref root); + + public: + HashmapIterator() = default; + HashmapIterator(Ref root, bool down = false) : down_(down) { + unwind(std::move(root)); + } + const Hashmap& operator*() const { + return *cur_; + } + const Hashmap* operator->() const { + return cur_.get(); + } + bool eof() { + return cur_.is_null(); + } + bool next(); + bool operator<(const HashmapIterator& other) const { + if (other.cur_.is_null()) { + return cur_.not_null(); + } else if (cur_.is_null()) { + return false; + } else { + return cur_->key().cmp(other.cur_->key()) * (down_ ? -1 : 1) < 0; + } + } + bool operator==(const HashmapIterator& other) const { + return other.cur_.is_null() ? cur_.is_null() : (cur_.not_null() && cur_->key() == other.cur_->key()); + } + bool operator!=(const HashmapIterator& other) const { + return other.cur_.is_null() ? cur_.not_null() : (cur_.is_null() || cur_->key() != other.cur_->key()); + } + HashmapIterator& operator++() { + next(); + return *this; + } +}; + +struct HashmapKeeper { + Ref root; + HashmapKeeper() = default; + HashmapKeeper(Ref _root) : root(std::move(_root)) { + } + Ref extract() { + return std::move(root); + } + operator Ref() const& { + return root; + } + operator Ref() && { + return std::move(root); + } + template + HashmapIdx operator[](Args&&... args) { + return HashmapIdx{root, DictKey{std::forward(args)...}}; + } + template + vm::StackEntry operator[](Args&&... args) const { + return Hashmap::get(root, DictKey{std::forward(args)...}); + } + vm::StackEntry get_key(const DictKey& key) const { + return Hashmap::get(root, key); + } + template + vm::StackEntry get(Args&&... args) const { + return Hashmap::get(root, DictKey{std::forward(args)...}); + } + vm::StackEntry get_remove_key(const DictKey& key) { + auto res = Hashmap::get_remove_key(root, key); + root = std::move(res.first); + return std::move(res.second); + } + template + vm::StackEntry get_remove(Args&&... args) { + return get_remove_key(DictKey{std::forward(args)...}); + } + bool remove_key(const DictKey& key) { + auto res = Hashmap::get_remove(root, key); + root = std::move(res.first); + return !res.second.is_null(); + } + template + bool remove(Args&&... args) { + return remove_key(DictKey{std::forward(args)...}); + } + template + void set(T key, vm::StackEntry value) { + root = Hashmap::set(root, DictKey(key), std::move(value)); + } + template + bool replace(T key, vm::StackEntry value) { + return Hashmap::replace(root, DictKey(key), std::move(value)); + } + HashmapIterator begin(bool reverse = false) const { + return HashmapIterator{root, reverse}; + } + HashmapIterator end() const { + return HashmapIterator{}; + } + HashmapIterator rbegin() const { + return HashmapIterator{root, true}; + } + HashmapIterator rend() const { + return HashmapIterator{}; + } +}; + +} // namespace fift diff --git a/crypto/fift/IntCtx.cpp b/crypto/fift/IntCtx.cpp index 9d623b06..2cac8849 100644 --- a/crypto/fift/IntCtx.cpp +++ b/crypto/fift/IntCtx.cpp @@ -20,7 +20,7 @@ namespace fift { -td::StringBuilder& operator<<(td::StringBuilder& os, const IntCtx& ctx) { +td::StringBuilder& operator<<(td::StringBuilder& os, const ParseCtx& ctx) { if (ctx.include_depth) { return os << ctx.filename << ":" << ctx.line_no << ": "; } else { @@ -28,7 +28,7 @@ td::StringBuilder& operator<<(td::StringBuilder& os, const IntCtx& ctx) { } } -std::ostream& operator<<(std::ostream& os, const IntCtx& ctx) { +std::ostream& operator<<(std::ostream& os, const ParseCtx& ctx) { return os << (PSLICE() << ctx).c_str(); } @@ -67,73 +67,7 @@ void CharClassifier::set_char_class(int c, int cl) { *p = static_cast((*p & ~mask) | cl); } -IntCtx::Savepoint::Savepoint(IntCtx& _ctx, std::string new_filename, std::string new_current_dir, - std::unique_ptr new_input_stream) - : ctx(_ctx) - , old_line_no(_ctx.line_no) - , old_need_line(_ctx.need_line) - , old_filename(_ctx.filename) - , old_current_dir(_ctx.currentd_dir) - , old_input_stream(_ctx.input_stream) - , old_input_stream_holder(std::move(_ctx.input_stream_holder)) - , old_curline(_ctx.str) - , old_curpos(_ctx.input_ptr - _ctx.str.c_str()) - , old_word(_ctx.word) { - ctx.line_no = 0; - ctx.filename = new_filename; - ctx.currentd_dir = new_current_dir; - ctx.input_stream = new_input_stream.get(); - ctx.input_stream_holder = std::move(new_input_stream); - ctx.str = ""; - ctx.input_ptr = 0; - ++(ctx.include_depth); -} - -bool IntCtx::Savepoint::restore(IntCtx& _ctx) { - if (restored || &ctx != &_ctx) { - return false; - } - ctx.line_no = old_line_no; - ctx.need_line = old_need_line; - ctx.filename = old_filename; - ctx.currentd_dir = old_current_dir; - ctx.input_stream = old_input_stream; - ctx.input_stream_holder = std::move(old_input_stream_holder); - ctx.str = old_curline; - ctx.input_ptr = ctx.str.c_str() + old_curpos; - ctx.word = old_word; - --(ctx.include_depth); - return restored = true; -} - -bool IntCtx::enter_ctx(std::string new_filename, std::string new_current_dir, - std::unique_ptr new_input_stream) { - if (!new_input_stream) { - return false; - } - ctx_save_stack.emplace_back(*this, std::move(new_filename), std::move(new_current_dir), std::move(new_input_stream)); - return true; -} - -bool IntCtx::leave_ctx() { - if (ctx_save_stack.empty()) { - return false; - } - bool ok = ctx_save_stack.back().restore(*this); - ctx_save_stack.pop_back(); - return ok; -} - -bool IntCtx::top_ctx() { - while (!ctx_save_stack.empty()) { - if (!leave_ctx()) { - return false; - } - } - return true; -} - -bool IntCtx::load_next_line() { +bool ParseCtx::load_next_line() { if (!std::getline(*input_stream, str)) { return false; } @@ -145,11 +79,11 @@ bool IntCtx::load_next_line() { return true; } -bool IntCtx::is_sb() const { +bool ParseCtx::is_sb() const { return !eof() && line_no == 1 && *input_ptr == '#' && input_ptr[1] == '!'; } -td::Slice IntCtx::scan_word_to(char delim, bool err_endl) { +td::Slice ParseCtx::scan_word_to(char delim, bool err_endl) { load_next_line_ifreq(); auto ptr = input_ptr; while (*ptr && *ptr != delim) { @@ -167,7 +101,7 @@ td::Slice IntCtx::scan_word_to(char delim, bool err_endl) { } } -td::Slice IntCtx::scan_word() { +td::Slice ParseCtx::scan_word() { skipspc(true); auto ptr = input_ptr; while (*ptr && *ptr != ' ' && *ptr != '\t' && *ptr != '\r') { @@ -179,7 +113,7 @@ td::Slice IntCtx::scan_word() { return td::Slice{ptr, ptr2}; } -td::Slice IntCtx::scan_word_ext(const CharClassifier& classifier) { +td::Slice ParseCtx::scan_word_ext(const CharClassifier& classifier) { skipspc(true); auto ptr = input_ptr; while (*ptr && *ptr != '\r' && *ptr != '\n') { @@ -196,7 +130,7 @@ td::Slice IntCtx::scan_word_ext(const CharClassifier& classifier) { return td::Slice{ptr, input_ptr}; } -void IntCtx::skipspc(bool skip_eol) { +void ParseCtx::skipspc(bool skip_eol) { do { while (*input_ptr == ' ' || *input_ptr == '\t' || *input_ptr == '\r') { ++input_ptr; @@ -207,6 +141,45 @@ void IntCtx::skipspc(bool skip_eol) { } while (load_next_line()); } +bool IntCtx::enter_ctx(std::unique_ptr new_parser) { + if (!new_parser) { + return false; + } + if (parser) { + parser_save_stack.push_back(std::move(parser)); + } + parser = std::move(new_parser); + return true; +} + +bool IntCtx::enter_ctx(std::string new_filename, std::string new_current_dir, + std::unique_ptr new_input_stream) { + if (!new_input_stream) { + return false; + } else { + return enter_ctx( + std::make_unique(std::move(new_input_stream), new_filename, new_current_dir, include_depth() + 1)); + } +} + +bool IntCtx::leave_ctx() { + if (parser_save_stack.empty()) { + return false; + } else { + parser = std::move(parser_save_stack.back()); + parser_save_stack.pop_back(); + return true; + } +} + +bool IntCtx::top_ctx() { + if (!parser_save_stack.empty()) { + parser = std::move(parser_save_stack[0]); + parser_save_stack.clear(); + } + return true; +} + void IntCtx::check_compile() const { if (state <= 0) { throw IntError{"compilation mode only"}; @@ -283,15 +256,20 @@ td::Result IntCtx::get_result() { } } +std::ostream& ParseCtx::show_context(std::ostream& os) const { + if (include_depth && line_no) { + os << filename << ":" << line_no << ":\t"; + } + if (!word.empty()) { + os << word << ":"; + } + return os; +} + td::Status IntCtx::add_error_loc(td::Status err) const { - if (err.is_error()) { + if (err.is_error() && parser) { std::ostringstream os; - if (include_depth && line_no) { - os << filename << ":" << line_no << ":\t"; - } - if (!word.empty()) { - os << word << ":"; - } + parser->show_context(os); return err.move_as_error_prefix(os.str()); } else { return err; diff --git a/crypto/fift/IntCtx.h b/crypto/fift/IntCtx.h index 807a5d01..5d574ae5 100644 --- a/crypto/fift/IntCtx.h +++ b/crypto/fift/IntCtx.h @@ -18,8 +18,8 @@ */ #pragma once -#include "crypto/vm/db/TonDb.h" // FIXME #include "crypto/vm/stack.hpp" +#include "crypto/vm/box.hpp" #include "crypto/common/bitstring.h" #include "td/utils/Status.h" @@ -32,6 +32,11 @@ #include #include +namespace vm { +class TonDbImpl; // from crypto/vm/db/TonDb.h +using TonDb = std::unique_ptr; +} // namespace vm + namespace fift { class Dictionary; class SourceLookup; @@ -68,71 +73,36 @@ class CharClassifier { } }; -struct IntCtx { - vm::Stack stack; - Ref next, exc_handler; - Ref exc_cont, exc_next; - int state{0}; +struct ParseCtx { int include_depth{0}; int line_no{0}; - int exit_code{0}; - td::Status error; bool need_line{true}; std::string filename; std::string currentd_dir; std::istream* input_stream{nullptr}; std::unique_ptr input_stream_holder; - std::ostream* output_stream{nullptr}; - std::ostream* error_stream{nullptr}; - - vm::TonDb* ton_db{nullptr}; - Dictionary* dictionary{nullptr}; - SourceLookup* source_lookup{nullptr}; - int* now{nullptr}; std::string word; private: std::string str; const char* input_ptr = nullptr; - class Savepoint { - IntCtx& ctx; - int old_line_no; - bool old_need_line; - bool restored{false}; - std::string old_filename; - std::string old_current_dir; - std::istream* old_input_stream; - std::unique_ptr old_input_stream_holder; - std::string old_curline; - std::ptrdiff_t old_curpos; - std::string old_word; - - public: - Savepoint(IntCtx& _ctx, std::string new_filename, std::string new_current_dir, - std::unique_ptr new_input_stream); - bool restore(IntCtx& _ctx); - }; - - std::vector ctx_save_stack; - public: - IntCtx() = default; - IntCtx(std::istream& _istream, std::string _filename, std::string _curdir = "", int _depth = 0) + ParseCtx() = default; + ParseCtx(std::istream& _istream, std::string _filename, std::string _curdir = "", int _depth = 0) : include_depth(_depth) , filename(std::move(_filename)) , currentd_dir(std::move(_curdir)) , input_stream(&_istream) { } - - operator vm::Stack&() { - return stack; + ParseCtx(std::unique_ptr _istream_ptr, std::string _filename, std::string _curdir = "", int _depth = 0) + : include_depth(_depth) + , filename(std::move(_filename)) + , currentd_dir(std::move(_curdir)) + , input_stream(_istream_ptr.get()) + , input_stream_holder(std::move(_istream_ptr)) { } - bool enter_ctx(std::string new_filename, std::string new_current_dir, std::unique_ptr new_input_stream); - bool leave_ctx(); - bool top_ctx(); - td::Slice scan_word_to(char delim, bool err_endl = true); td::Slice scan_word(); td::Slice scan_word_ext(const CharClassifier& classifier); @@ -165,6 +135,50 @@ struct IntCtx { bool is_sb() const; + std::ostream& show_context(std::ostream& os) const; +}; + +struct IntCtx { + vm::Stack stack; + Ref next, exc_handler; + Ref exc_cont, exc_next; + int state{0}; + int exit_code{0}; + td::Status error; + + std::unique_ptr parser; + std::vector> parser_save_stack; + + std::ostream* output_stream{nullptr}; // move to OutCtx? + std::ostream* error_stream{nullptr}; + + vm::TonDb* ton_db{nullptr}; + SourceLookup* source_lookup{nullptr}; + int* now{nullptr}; + + Dictionary dictionary, main_dictionary, context; + + public: + IntCtx() = default; + IntCtx(std::istream& _istream, std::string _filename, std::string _curdir = "", int _depth = 0) { + parser = std::make_unique(_istream, _filename, _curdir, _depth); + } + IntCtx(std::unique_ptr _istream, std::string _filename, std::string _curdir = "", int _depth = 0) { + parser = std::make_unique(std::move(_istream), _filename, _curdir, _depth); + } + + bool enter_ctx(std::unique_ptr new_ctx); + bool enter_ctx(std::string new_filename, std::string new_current_dir, std::unique_ptr new_input_stream); + bool leave_ctx(); + bool top_ctx(); + int include_depth() const { + return parser ? parser->include_depth : -1; + } + + operator vm::Stack &() { + return stack; + } + void clear() { state = 0; stack.clear(); @@ -194,6 +208,6 @@ struct IntCtx { td::Result run(Ref cont); }; -td::StringBuilder& operator<<(td::StringBuilder& os, const IntCtx& ctx); -std::ostream& operator<<(std::ostream& os, const IntCtx& ctx); +td::StringBuilder& operator<<(td::StringBuilder& os, const ParseCtx& ctx); +std::ostream& operator<<(std::ostream& os, const ParseCtx& ctx); } // namespace fift diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 01fb8dd0..85045f29 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -1,12 +1,16 @@ library TVM_Asm // simple TVM Assembler +namespace Asm +Asm definitions +"0.4.3" constant asm-fif-version + variable @atend variable @was-split false @was-split ! { "not in asm context" abort } @atend ! { `normal eq? not abort"must be terminated by }>" } : @normal? -{ @atend @ 1 { @atend ! @normal? } does @atend ! } : @pushatend -{ @pushatend { }> b> } : }>c @@ -251,7 +255,7 @@ x{7F} @Defop TRUE } cond } cond @addopb } dup : PUSHINT : INT -{ = or abort"invalid slice padding" swap 1 1 u, 0 rot u, } : @scomplete -{ tuck sbitrefs swap 17 + swap @havebitrefs not +{ tuck sbitrefs swap 26 + swap @havebitrefs not { PUSHREFSLICE } { over sbitrefs 2dup 123 0 2x<= { drop tuck 4 + 3 >> swap x{8B} s, over 4 u, 3 roll s, @@ -324,38 +328,85 @@ x{A7} x{A8} @Defop(8i,alt) MULCONST ' SUBCONST : SUBINT ' MULCONST : MULINT x{A8} @Defop MUL + x{A904} @Defop DIV x{A905} @Defop DIVR x{A906} @Defop DIVC x{A908} @Defop MOD +x{A909} @Defop MODR +x{A90A} @Defop MODC x{A90C} @Defop DIVMOD x{A90D} @Defop DIVMODR x{A90E} @Defop DIVMODC + x{A925} @Defop RSHIFTR x{A926} @Defop RSHIFTC +x{A928} @Defop MODPOW2 +x{A929} @Defop MODPOW2R +x{A92A} @Defop MODPOW2C +x{A92C} @Defop RSHIFTMOD +x{A92D} @Defop RSHIFTMODR +x{A92E} @Defop RSHIFTMODC + x{A935} @Defop(8u+1) RSHIFTR# x{A936} @Defop(8u+1) RSHIFTC# x{A938} @Defop(8u+1) MODPOW2# x{A939} @Defop(8u+1) MODPOW2R# x{A93A} @Defop(8u+1) MODPOW2C# +x{A93C} @Defop(8u+1) RSHIFT#MOD +x{A93D} @Defop(8u+1) RSHIFTR#MOD +x{A93E} @Defop(8u+1) RSHIFTC#MOD + x{A984} @Defop MULDIV x{A985} @Defop MULDIVR +x{A986} @Defop MULDIVC x{A988} @Defop MULMOD +x{A989} @Defop MULMODR +x{A98A} @Defop MULMODC x{A98C} @Defop MULDIVMOD x{A98D} @Defop MULDIVMODR x{A98E} @Defop MULDIVMODC + x{A9A4} @Defop MULRSHIFT x{A9A5} @Defop MULRSHIFTR x{A9A6} @Defop MULRSHIFTC +x{A9A8} @Defop MULMODPOW2 +x{A9A9} @Defop MULMODPOW2R +x{A9AA} @Defop MULMODPOW2C +x{A9AC} @Defop MULRSHIFTMOD +x{A9AD} @Defop MULRSHIFTRMOD +x{A9AE} @Defop MULRSHIFTCMOD + x{A9B4} @Defop(8u+1) MULRSHIFT# x{A9B5} @Defop(8u+1) MULRSHIFTR# x{A9B6} @Defop(8u+1) MULRSHIFTC# +x{A9B8} @Defop(8u+1) MULMODPOW2# +x{A9B9} @Defop(8u+1) MULMODPOW2R# +x{A9BA} @Defop(8u+1) MULMODPOW2C# +x{A9BC} @Defop(8u+1) MULRSHIFT#MOD +x{A9BD} @Defop(8u+1) MULRSHIFTR#MOD +x{A9BE} @Defop(8u+1) MULRSHIFTC#MOD + x{A9C4} @Defop LSHIFTDIV x{A9C5} @Defop LSHIFTDIVR x{A9C6} @Defop LSHIFTDIVC +x{A9C8} @Defop LSHIFTMOD +x{A9C9} @Defop LSHIFTMODR +x{A9CA} @Defop LSHIFTMODC +x{A9CC} @Defop LSHIFTDIVMOD +x{A9CD} @Defop LSHIFTDIVMODR +x{A9CE} @Defop LSHIFTDIVMODC + x{A9D4} @Defop(8u+1) LSHIFT#DIV x{A9D5} @Defop(8u+1) LSHIFT#DIVR x{A9D6} @Defop(8u+1) LSHIFT#DIVC +x{A9D8} @Defop(8u+1) LSHIFT#MOD +x{A9D9} @Defop(8u+1) LSHIFT#MODR +x{A9DA} @Defop(8u+1) LSHIFT#MODC +x{A9DC} @Defop(8u+1) LSHIFT#DIVMOD +x{A9DD} @Defop(8u+1) LSHIFT#DIVMODR +x{A9DE} @Defop(8u+1) LSHIFT#DIVMODC + x{AA} @Defop(8u+1) LSHIFT# x{AB} @Defop(8u+1) RSHIFT# x{AC} @Defop LSHIFT @@ -504,6 +555,7 @@ x{CF1E} @Defop STSLICERQ x{CF1F} dup @Defop STBRQ @Defop BCONCATQ x{CF20} @Defop(ref) STREFCONST { } : }END> @@ -1319,3 +1388,44 @@ forget @proclist forget @proccnt { spec } : hash>libref // ( c -- c' ) { hash hash>libref } : >libref + +{ dup "." $pos dup -1 = + { drop 0 } + { $| 1 $| nip swap (number) 1- abort"invalid version" + dup dup 0 < swap 999 > or abort"invalid version" + } + cond +} : parse-version-level + +{ + 0 swap + "." $+ + { swap 1000 * swap parse-version-level rot + swap } 3 times + "" $= not abort"invalid version" +} : parse-asm-fif-version + +{ + dup =: required-version parse-asm-fif-version + asm-fif-version parse-asm-fif-version + = 1+ { + "Required Asm.fif version: " @' required-version "; actual Asm.fif version: " asm-fif-version $+ $+ $+ abort + } if +} : require-asm-fif-version + +{ + dup =: required-version parse-asm-fif-version + asm-fif-version parse-asm-fif-version + swap + >= 1+ { + "Required Asm.fif version: " @' required-version "; actual Asm.fif version: " asm-fif-version $+ $+ $+ abort + } if +} : require-asm-fif-version>= + + +Fift definitions Asm +' <{ : <{ +' PROGRAM{ : PROGRAM{ +' asm-fif-version : asm-fif-version +' require-asm-fif-version : require-asm-fif-version +' require-asm-fif-version>= : require-asm-fif-version>= +Fift diff --git a/crypto/fift/lib/Disasm.fif b/crypto/fift/lib/Disasm.fif new file mode 100644 index 00000000..a46eb5b2 --- /dev/null +++ b/crypto/fift/lib/Disasm.fif @@ -0,0 +1,141 @@ +library TVM_Disasm +// simple TVM Disassembler +"Lists.fif" include + +variable 'disasm +{ 'disasm @ execute } : disasm // disassemble a slice +// usage: x{74B0} disasm + +variable @dismode @dismode 0! +{ rot over @ and rot xor swap ! } : andxor! +{ -2 0 @dismode andxor! } : stack-disasm // output 's1 s4 XCHG' +{ -2 1 @dismode andxor! } : std-disasm // output 'XCHG s1, s4' +{ -3 2 @dismode andxor! } : show-vm-code +{ -3 0 @dismode andxor! } : hide-vm-code +{ @dismode @ 1 and 0= } : stack-disasm? + +variable @indent @indent 0! +{ ' space @indent @ 2* times } : .indent +{ @indent 1+! } : +indent +{ @indent 1-! } : -indent + +{ " " $pos } : spc-pos +{ dup " " $pos swap "," $pos dup 0< { drop } { + over 0< { nip } { min } cond } cond +} : spc-comma-pos +{ { dup spc-pos 0= } { 1 $| nip } while } : -leading +{ -leading -trailing dup spc-pos dup 0< { + drop dup $len { atom single } { drop nil } cond } { + $| swap atom swap -leading 2 { over spc-comma-pos dup 0>= } { + swap 1+ -rot $| 1 $| nip -leading rot + } while drop tuple + } cond +} : parse-op +{ dup "s-1" $= { drop "s(-1)" true } { + dup "s-2" $= { drop "s(-2)" true } { + dup 1 $| swap "x" $= { nip "x{" swap $+ +"}" true } { + 2drop false } cond } cond } cond +} : adj-op-arg +{ over count over <= { drop } { 2dup [] adj-op-arg { swap []= } { drop } cond } cond } : adj-arg[] +{ 1 adj-arg[] 2 adj-arg[] 3 adj-arg[] + dup first + dup `XCHG eq? { + drop dup count 2 = { tpop swap "s0" , swap , } if } { + dup `LSHIFT eq? { + drop dup count 2 = stack-disasm? and { second `LSHIFT# swap pair } if } { + dup `RSHIFT eq? { + drop dup count 2 = stack-disasm? and { second `RSHIFT# swap pair } if } { + drop + } cond } cond } cond +} : adjust-op + +variable @cp @cp 0! +variable @curop +variable @contX variable @contY variable @cdict + +{ atom>$ type } : .atom +{ dup first .atom dup count 1 > { space 0 over count 2- { 1+ 2dup [] type .", " } swap times 1+ [] type } { drop } cond } : std-show-op +{ 0 over count 1- { 1+ 2dup [] type space } swap times drop first .atom } : stk-show-op +{ @dismode @ 2 and { .indent ."// " @curop @ csr. } if } : .curop? +{ .curop? .indent @dismode @ 1 and ' std-show-op ' stk-show-op cond cr +} : show-simple-op +{ dup 4 u@ 9 = { 8 u@+ swap 15 and 3 << s@ } { + dup 7 u@ 0x47 = { 7 u@+ nip 2 u@+ 7 u@+ -rot 3 << swap sr@ } { + dup 8 u@ 0x8A = { ref@ " cr } : show-cont-op +{ swap scont-swap ":<{" show-cont-bodyx scont-swap + "" show-cont-bodyx .indent ."}>" cr } : show-cont2-op + +{ @contX @ null? { "CONT" show-cont-op } ifnot +} : flush-contX +{ @contY @ null? { scont-swap "CONT" show-cont-op scont-swap } ifnot +} : flush-contY +{ flush-contY flush-contX } : flush-cont +{ @contX @ null? not } : have-cont? +{ @contY @ null? not } : have-cont2? +{ flush-contY @contY ! scont-swap } : save-cont-body + +{ @cdict ! } : save-const-dict +{ @cdict null! } : flush-dict +{ @cdict @ null? not } : have-dict? + +{ flush-cont .indent type .":<{" cr + @curop @ ref@ " cr +} : show-ref-op +{ flush-contY .indent rot type .":<{" cr + @curop @ ref@ " cr +} : show-cont-ref-op +{ flush-cont .indent swap type .":<{" cr + @curop @ ref@+ " cr +} : show-ref2-op + +{ flush-cont first atom>$ dup 5 $| drop "DICTI" $= swap + .indent type ." {" cr +indent @cdict @ @cdict null! unpair + rot { + swap .indent . ."=> <{" cr +indent disasm -indent .indent ."}>" cr true + } swap ' idictforeach ' dictforeach cond drop + -indent .indent ."}" cr +} : show-const-dict-op + +( `PUSHCONT `PUSHREFCONT ) constant @PushContL +( `REPEAT `UNTIL `IF `IFNOT `IFJMP `IFNOTJMP ) constant @CmdC1 +( `IFREF `IFNOTREF `IFJMPREF `IFNOTJMPREF `CALLREF `JMPREF ) constant @CmdR1 +( `DICTIGETJMP `DICTIGETJMPZ `DICTUGETJMP `DICTUGETJMPZ `DICTIGETEXEC `DICTUGETEXEC ) constant @JmpDictL +{ dup first `DICTPUSHCONST eq? { + flush-cont @curop @ get-const-dict save-const-dict show-simple-op } { + dup first @JmpDictL list-member? have-dict? and { + flush-cont show-const-dict-op } { + flush-dict + dup first @PushContL list-member? { + drop @curop @ get-cont-body save-cont-body } { + dup first @CmdC1 list-member? have-cont? and { + flush-contY first atom>$ .curop? show-cont-op } { + dup first @CmdR1 list-member? { + flush-cont first atom>$ dup $len 3 - $| drop .curop? show-ref-op } { + dup first `WHILE eq? have-cont2? and { + drop "WHILE" "}>DO<{" .curop? show-cont2-op } { + dup first `IFELSE eq? have-cont2? and { + drop "IF" "}>ELSE<{" .curop? show-cont2-op } { + dup first dup `IFREFELSE eq? swap `IFELSEREF eq? or have-cont? and { + first `IFREFELSE eq? "IF" "}>ELSE<{" rot .curop? show-cont-ref-op } { + dup first `IFREFELSEREF eq? { + drop "IF" "}>ELSE<{" .curop? show-ref2-op } { + flush-cont show-simple-op + } cond } cond } cond } cond } cond } cond } cond } cond } cond +} : show-op +{ dup @cp @ (vmoplen) dup 0> { 65536 /mod swap sr@+ swap dup @cp @ (vmopdump) parse-op swap s> true } { drop false } cond } : fetch-one-op +{ { fetch-one-op } { swap @curop ! adjust-op show-op } while } : disasm-slice +{ { disasm-slice dup sbitrefs 1- or 0= } { ref@ = -rot <= and } : s-fits? diff --git a/crypto/fift/lib/FiftExt.fif b/crypto/fift/lib/FiftExt.fif new file mode 100644 index 00000000..6ed677d7 --- /dev/null +++ b/crypto/fift/lib/FiftExt.fif @@ -0,0 +1,118 @@ +{ ?dup { 1+ { execute } { 0 swap } cond } + { (number) ?dup 0= abort"-?" 'nop } cond +} : (interpret-prepare) +{ { include-depth 0= (seekeof?) not } { + (word-prefix-find) (interpret-prepare) (execute) + } while +} : interpret +{ ({) + { 0 (seekeof?) abort"no }" (word-prefix-find) (interpret-prepare) (compile) over atom? not } until + (}) swap execute +} : begin-block +{ swap 0 'nop } : end-block +{ { 1 'nop } `{ begin-block } +{ { swap `{ eq? not abort"} without {" swap execute } end-block } +:: } :: { + +// if{ ... }then{ ... }elseif{ ... }then{ ... }else{ ... } +{ eq? not abort"unexpected" } : ?pairs +{ dup `if eq? swap `ifnot eq? over or not abort"without if{" } : if-ifnot? +// cond then ? -- exec +{ { ' if } { ' ifnot } cond rot ({) 0 rot (compile) -rot 1 swap (compile) (}) +} : (make-if) +// cond then else -- exec +{ rot ({) 0 rot (compile) -rot 2 ' cond (compile) (}) +} : (make-cond) +{ `noelse `if begin-block } :: if{ +{ `noelse `ifnot begin-block } :: ifnot{ +{ 1 ' end-block does } : end-block-does +{ { over `else eq? } { + nip rot if-ifnot? ' swap ifnot (make-cond) + } while + swap `noelse ?pairs 0 swap +} : finish-else-chain +{ swap dup if-ifnot? drop `then { + swap `then ?pairs + swap if-ifnot? (make-if) finish-else-chain + } `{ begin-block +} end-block-does :: }then{ +{ swap `{ ?pairs nip + swap `then eq? not abort"without }then{" `else +} : ?else-ok +{ ?else-ok { finish-else-chain } `{ begin-block } end-block-does :: }else{ +{ ?else-ok `if begin-block } end-block-does :: }elseif{ +{ ?else-ok `ifnot begin-block } end-block-does :: }elseifnot{ + +// while{ ... }do{ ... } +{ 2 ' while does } : (make-while) +{ `while begin-block } :: while{ +{ swap `while eq? not abort"without while{" `while-do { + swap `while-do ?pairs (make-while) 0 swap + } `{ begin-block +} end-block-does :: }do{ + +// repeat{ ... }until{ ... } +{ swap ({) 0 rot (compile) 0 rot (compile) (}) 1 ' until does } : (make-until) +{ `repeat begin-block } :: repeat{ +{ swap `repeat eq? not abort"without repeat{" `until { + swap `until ?pairs (make-until) 0 swap + } `{ begin-block +} end-block-does :: }until{ + +// def { ... } instead of { ... } : +{ bl word swap bl word "{" $cmp abort"{ expected" `def { + swap `def ?pairs -rot 3 ' (create) + } `{ begin-block +} : (def) +{ 0 (def) } :: def +{ 1 (def) } :: def:: + +// defrec { ... } instead of recursive { ... } swap ! +{ recursive bl word "{" $cmp abort"{ expected" `defrec { + swap `defrec ?pairs swap ! 0 'nop + } `{ begin-block +} :: defrec + +def .sgn { + if{ ?dup 0= }then{ + ."zero" + }elseif{ 0> }then{ + ."positive" + }else{ + ."negative" + } + cr +} +// equivalent to: { ?dup 0= { ."zero" } { 0> { ."positive" } { ."negative" } cond } cond cr } : .sgn + +defrec fact { + if{ dup }then{ + dup 1- fact * + }else{ + drop 1 + } +} +// equivalent to: recursive fact { dup { dup 1- fact * } { drop 1 } cond } swap ! + +// [[ ... ]] computes arbitrary constants inside definitions +// { [[ 5 dup * ]] + } : add25 +// is equivalent to +// { 25 + } : add25 +{ "without [[" abort } box constant ']] +{ ']] @ execute } : ]] +{ { ']] @ 2 { ']] ! call/cc } does ']] ! + interpret 'nop ']] ! "]] not found" abort + } call/cc + drop 1 'nop +} :: [[ + +{ { over @ swap 2 { call/cc } does swap ! + interpret "literal to eof" abort + } call/cc + drop execute 1 'nop +} : interpret-literal-to +// use next line only if Lists.fif is loaded (or move it to Lists.fif if FiftExt.fif becomes part of Fift.fif) +// { ( ') interpret-literal-to } :: '( +// then you can use list literals '( a b c ... ) inside definitions: +// { '( 1 2 3 ) } : test +// { '( ( `a { ."A" } ) ( `b { ."B" } ) ) assoc { cadr execute } { ."???" } cond } : test2 diff --git a/crypto/fift/utils.cpp b/crypto/fift/utils.cpp index b8c5c61f..68fc18c0 100644 --- a/crypto/fift/utils.cpp +++ b/crypto/fift/utils.cpp @@ -59,6 +59,12 @@ td::Result load_GetOpt_fif(std::string dir = "") { td::Result load_wallet3_code_fif(std::string dir = "") { return td::read_file_str(smartcont_dir(dir) + "wallet-v3-code.fif"); } +td::Result load_FiftExt_fif(std::string dir = "") { + return load_source("FiftExt.fif", dir); +} +td::Result load_Disasm_fif(std::string dir = "") { + return load_source("Disasm.fif", dir); +} class MemoryFileLoader : public fift::FileLoader { public: @@ -108,7 +114,8 @@ class MemoryFileLoader : public fift::FileLoader { td::Result create_source_lookup(std::string main, bool need_preamble = true, bool need_asm = true, bool need_ton_util = true, bool need_lisp = true, - bool need_w3_code = true, std::string dir = "") { + bool need_w3_code = true, bool need_fift_ext = true, + bool need_disasm = true, std::string dir = "") { auto loader = std::make_unique(); loader->add_file("/main.fif", std::move(main)); if (need_preamble) { @@ -141,6 +148,14 @@ td::Result create_source_lookup(std::string main, bool need_ TRY_RESULT(f, load_wallet3_code_fif(dir)); loader->add_file("/wallet-v3-code.fif", std::move(f)); } + if (need_fift_ext) { + TRY_RESULT(f, load_FiftExt_fif(dir)); + loader->add_file("/FiftExt.fif", std::move(f)); + } + if (need_disasm) { + TRY_RESULT(f, load_Disasm_fif(dir)); + loader->add_file("/Disasm.fif", std::move(f)); + } auto res = fift::SourceLookup(std::move(loader)); res.add_include_path("/"); return std::move(res); @@ -172,7 +187,7 @@ td::Result run_fift(fift::SourceLookup source_lookup, std::o } // namespace td::Result mem_run_fift(std::string source, std::vector args, std::string fift_dir) { std::stringstream ss; - TRY_RESULT(source_lookup, create_source_lookup(source, true, true, true, true, true, fift_dir)); + TRY_RESULT(source_lookup, create_source_lookup(source, true, true, true, true, true, true, true, fift_dir)); TRY_RESULT_ASSIGN(source_lookup, run_fift(std::move(source_lookup), &ss, true, std::move(args))); FiftOutput res; res.source_lookup = std::move(source_lookup); @@ -190,7 +205,8 @@ td::Result mem_run_fift(SourceLookup source_lookup, std::vector create_mem_source_lookup(std::string main, std::string fift_dir, bool need_preamble, bool need_asm, bool need_ton_util, bool need_lisp, bool need_w3_code) { - return create_source_lookup(main, need_preamble, need_asm, need_ton_util, need_lisp, need_w3_code, fift_dir); + return create_source_lookup(main, need_preamble, need_asm, need_ton_util, need_lisp, need_w3_code, false, false, + fift_dir); } td::Result> compile_asm(td::Slice asm_code, std::string fift_dir, bool is_raw) { @@ -198,7 +214,7 @@ td::Result> compile_asm(td::Slice asm_code, std::string fift_d TRY_RESULT(source_lookup, create_source_lookup(PSTRING() << "\"Asm.fif\" include\n " << (is_raw ? "<{" : "") << asm_code << "\n" << (is_raw ? "}>c" : "") << " boc>B \"res\" B>file", - true, true, true, false, false, fift_dir)); + true, true, true, false, false, false, false, fift_dir)); TRY_RESULT(res, run_fift(std::move(source_lookup), &ss)); TRY_RESULT(boc, res.read_file("res")); return vm::std_boc_deserialize(std::move(boc.data)); diff --git a/crypto/fift/words.cpp b/crypto/fift/words.cpp index 7556a127..964b4328 100644 --- a/crypto/fift/words.cpp +++ b/crypto/fift/words.cpp @@ -21,6 +21,7 @@ #include "Dictionary.h" #include "IntCtx.h" #include "SourceLookup.h" +#include "HashMap.h" #include "common/refcnt.hpp" #include "common/bigint.hpp" @@ -42,6 +43,8 @@ #include "vm/box.hpp" #include "vm/atom.h" +#include "vm/db/TonDb.h" // only for interpret_db_run_vm{,_parallel} + #include "block/block.h" #include "td/utils/filesystem.h" @@ -58,12 +61,31 @@ namespace fift { -void show_total_cells(std::ostream& stream) { - stream << "total cells = " << vm::DataCell::get_total_data_cells() << std::endl; +const Ref nop_word_def = Ref{true}; + +// +// functions for wordef +// +Ref pop_exec_token(vm::Stack& stack) { + auto wd_ref = stack.pop_chk().as_object(); + if (wd_ref.is_null()) { + throw IntError{"execution token expected"}; + } + return wd_ref; } -void do_compile(vm::Stack& stack, Ref word_def); -void do_compile_literals(vm::Stack& stack, int count); +Ref pop_word_list(vm::Stack& stack) { + auto wl_ref = stack.pop_chk().as_object(); + if (wl_ref.is_null()) { + throw IntError{"word list expected"}; + } + return wl_ref; +} + +void push_argcount(vm::Stack& stack, int args) { + stack.push_smallint(args); + stack.push_object(nop_word_def); +} void interpret_dot(IntCtx& ctx, bool space_after) { *ctx.output_stream << dec_string2(ctx.stack.pop_int()) << (space_after ? " " : ""); @@ -121,7 +143,7 @@ void interpret_print_list(IntCtx& ctx) { } void interpret_dottc(IntCtx& ctx) { - show_total_cells(*ctx.output_stream); + *ctx.output_stream << "total cells = " << vm::DataCell::get_total_data_cells() << std::endl; } void interpret_dot_internal(vm::Stack& stack) { @@ -461,7 +483,7 @@ void interpret_make_xchg(vm::Stack& stack) { if (x) { stack.push_object(td::Ref{true, std::bind(interpret_xchg, _1, x, y)}); } else if (y <= 1) { - stack.push_object(y ? swap_word_def : Dictionary::nop_word_def); + stack.push_object(y ? swap_word_def : nop_word_def); } else { stack.push_object(td::Ref{true, std::bind(interpret_xchg0, _1, y)}); } @@ -1080,6 +1102,31 @@ void interpret_fetch_bytes(vm::Stack& stack, int mode) { } } +void interpret_fetch_slice(vm::Stack& stack, int mode) { + unsigned refs = ((mode & 1) ? stack.pop_smallint_range(4) : 0); + unsigned bits = stack.pop_smallint_range(1023); + auto cs = stack.pop_cellslice(); + if (!cs->have(bits, refs)) { + if (mode & 2) { + stack.push(std::move(cs)); + } + stack.push_bool(false); + if (!(mode & 4)) { + throw IntError{"end of data while fetching subslice from cell"}; + } + } else { + if (mode & 2) { + stack.push(cs.write().fetch_subslice(bits, refs)); + stack.push(std::move(cs)); + } else { + stack.push(cs->prefetch_subslice(bits, refs)); + } + if (mode & 4) { + stack.push_bool(true); + } + } +} + void interpret_cell_empty(vm::Stack& stack) { auto cs = stack.pop_cellslice(); stack.push_bool(cs->empty_ext()); @@ -1483,6 +1530,150 @@ void interpret_crc32c(vm::Stack& stack) { stack.push_smallint(td::crc32c(td::Slice{str})); } +// Fift hashmaps + +void push_hmap(vm::Stack& stack, Ref hmap) { + if (hmap.not_null()) { + stack.push_object(std::move(hmap)); + } else { + stack.push({}); + } +} + +void push_hmap(vm::Stack& stack, HashmapKeeper hmap_keep) { + push_hmap(stack, hmap_keep.extract()); +} + +Ref pop_hmap(vm::Stack& stack) { + stack.check_underflow(1); + auto se = stack.pop(); + if (se.is_null()) { + return {}; + } + auto hmap_ref = std::move(se).as_object(); + if (hmap_ref.is_null()) { + throw IntError{"hashmap expected"}; + } + return hmap_ref; +} + +HashmapKeeper pop_hmap_keeper(vm::Stack& stack) { + return HashmapKeeper{pop_hmap(stack)}; +} + +void interpret_hmap_new(vm::Stack& stack) { + stack.push({}); +} + +void interpret_hmap_fetch(vm::Stack& stack, int mode) { + auto hmap = pop_hmap(stack); + auto value = Hashmap::get(std::move(hmap), stack.pop_chk()); + bool found = !value.is_null(); + if ((mode & 8) && !found) { + throw IntError{"hashmap key not found"}; + } + if (mode & (2 << (int)found)) { + stack.push(std::move(value)); + } + if (mode & 1) { + stack.push_bool(found); + } +} + +void interpret_hmap_delete(vm::Stack& stack, int mode) { + auto hmap = pop_hmap(stack); + auto res = Hashmap::get_remove(std::move(hmap), stack.pop_chk()); + push_hmap(stack, std::move(res.first)); + bool found = !res.second.is_null(); + if ((mode & 8) && !found) { + throw IntError{"hashmap key not found"}; + } + if (mode & (2 << (int)found)) { + stack.push(std::move(res.second)); + } + if (mode & 1) { + stack.push_bool(found); + } +} + +void interpret_hmap_store(vm::Stack& stack, int mode) { + stack.check_underflow(3); + auto hmap = pop_hmap_keeper(stack); + auto key = stack.pop(), value = stack.pop(); + bool ok = true; + if (mode & 1) { + hmap.set(std::move(key), std::move(value)); + } else { + ok = hmap.replace(std::move(key), std::move(value)); + } + push_hmap(stack, std::move(hmap)); + if (mode & 2) { + stack.push_bool(ok); + } +} + +void interpret_hmap_is_empty(vm::Stack& stack) { + stack.push_bool(pop_hmap(stack).is_null()); +} + +void interpret_hmap_decompose(vm::Stack& stack, int mode) { + auto hmap = pop_hmap(stack); + if (hmap.is_null()) { + if (mode & 1) { + stack.push_bool(false); + } else { + throw IntError{"empty hmap"}; + } + return; + } + stack.push(hmap->key()); + stack.push(hmap->value()); + push_hmap(stack, hmap->left()); + push_hmap(stack, hmap->right()); + if (mode & 1) { + stack.push_bool(true); + } +} + +class HmapIterCont : public LoopCont { + HashmapIterator it; + bool ok; + + public: + HmapIterCont(Ref _func, Ref _after, HashmapIterator _it) + : LoopCont(std::move(_func), std::move(_after)), it(std::move(_it)), ok(true) { + } + HmapIterCont(const HmapIterCont&) = default; + HmapIterCont* make_copy() const override { + return new HmapIterCont(*this); + } + bool init(IntCtx& ctx) override { + return true; + } + bool pre_exec(IntCtx& ctx) override { + if (it.eof()) { + return false; + } else { + ctx.stack.push(it->key()); + ctx.stack.push(it->value()); + return true; + } + } + bool post_exec(IntCtx& ctx) override { + ok = ctx.stack.pop_bool(); + return ok && it.next(); + } + bool finalize(IntCtx& ctx) override { + ctx.stack.push_bool(ok); + return true; + } +}; + +Ref interpret_hmap_foreach(IntCtx& ctx, int mode) { + auto func = pop_exec_token(ctx); + return td::make_ref(std::move(func), std::move(ctx.next), pop_hmap_keeper(ctx).begin(mode & 1)); +} + // vm dictionaries void interpret_dict_new(vm::Stack& stack) { stack.push({}); @@ -1882,7 +2073,7 @@ void interpret_pfx_dict_get(vm::Stack& stack) { } void interpret_bitstring_hex_literal(IntCtx& ctx) { - auto s = ctx.scan_word_to('}'); + auto s = ctx.parser->scan_word_to('}'); unsigned char buff[128]; int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), s.begin(), s.end()); if (bits < 0) { @@ -1890,11 +2081,11 @@ void interpret_bitstring_hex_literal(IntCtx& ctx) { } auto cs = Ref{true, vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize()}; ctx.stack.push(std::move(cs)); - push_argcount(ctx.stack, 1); + push_argcount(ctx, 1); } void interpret_bitstring_binary_literal(IntCtx& ctx) { - auto s = ctx.scan_word_to('}'); + auto s = ctx.parser->scan_word_to('}'); unsigned char buff[128]; int bits = (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff), s.begin(), s.end()); if (bits < 0) { @@ -1902,12 +2093,12 @@ void interpret_bitstring_binary_literal(IntCtx& ctx) { } auto cs = Ref{true, vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize()}; ctx.stack.push(std::move(cs)); - push_argcount(ctx.stack, 1); + push_argcount(ctx, 1); } void interpret_word(IntCtx& ctx) { char sep = (char)ctx.stack.pop_smallint_range(127); - auto word = (sep != ' ' ? ctx.scan_word_to(sep, true) : ctx.scan_word()); + auto word = (sep != ' ' ? ctx.parser->scan_word_to(sep, true) : ctx.parser->scan_word()); ctx.stack.push_string(word); } @@ -1915,17 +2106,17 @@ void interpret_word_ext(IntCtx& ctx) { int mode = ctx.stack.pop_smallint_range(11); auto delims = ctx.stack.pop_string(); if (mode & 8) { - ctx.skipspc(mode & 4); + ctx.parser->skipspc(mode & 4); } - ctx.stack.push_string(ctx.scan_word_ext(CharClassifier{delims, mode & 3})); + ctx.stack.push_string(ctx.parser->scan_word_ext(CharClassifier{delims, mode & 3})); } void interpret_skipspc(IntCtx& ctx) { - ctx.skipspc(); + ctx.parser->skipspc(); } void interpret_wordlist_begin_aux(vm::Stack& stack) { - stack.push({vm::from_object, Ref{true}}); + stack.push_make_object(); } void interpret_wordlist_begin(IntCtx& ctx) { @@ -1938,7 +2129,7 @@ void interpret_wordlist_begin(IntCtx& ctx) { void interpret_wordlist_end_aux(vm::Stack& stack) { Ref wordlist_ref = pop_word_list(stack); wordlist_ref.write().close(); - stack.push({vm::from_object, Ref{wordlist_ref}}); + stack.push_object(std::move(wordlist_ref)); } void interpret_wordlist_end(IntCtx& ctx) { @@ -1957,7 +2148,7 @@ void interpret_internal_interpret_begin(IntCtx& ctx) { void interpret_internal_interpret_end(IntCtx& ctx) { ctx.check_int_exec(); ctx.state = -ctx.state; - ctx.stack.push({vm::from_object, Dictionary::nop_word_def}); + ctx.stack.push_object(nop_word_def); } // (create) @@ -1974,18 +2165,12 @@ void interpret_create_aux(IntCtx& ctx, int mode) { if (!(mode & 2)) { word += ' '; } - bool active = (mode & 1); - auto entry = ctx.dictionary->lookup(word); - if (entry) { - *entry = DictEntry{std::move(wd_ref), active}; // redefine word - } else { - ctx.dictionary->def_word(std::move(word), {std::move(wd_ref), active}); - } + ctx.dictionary.def_word(std::move(word), {std::move(wd_ref), (bool)(mode & 1)}); } // { bl word 0 (create) } : create void interpret_create(IntCtx& ctx) { - auto word = ctx.scan_word(); + auto word = ctx.parser->scan_word(); if (!word.size()) { throw IntError{"non-empty word name expected"}; } @@ -1997,10 +2182,10 @@ Ref create_aux_wd{Ref{true, std::bind(interpret_create_aux, s // { bl word 2 ' (create) } :: : void interpret_colon(IntCtx& ctx, int mode) { - ctx.stack.push_string(ctx.scan_word()); + ctx.stack.push_string(ctx.parser->scan_word()); ctx.stack.push_smallint(mode); ctx.stack.push_smallint(2); - ctx.stack.push({vm::from_object, create_aux_wd}); + ctx.stack.push_object(create_aux_wd); //push_argcount(ctx, 2, create_wd); } @@ -2008,27 +2193,27 @@ void interpret_colon(IntCtx& ctx, int mode) { void interpret_forget_aux(IntCtx& ctx) { std::string s = ctx.stack.pop_string(); auto s_copy = s; - auto entry = ctx.dictionary->lookup(s); + auto entry = ctx.dictionary.lookup(s); if (!entry) { s += " "; - entry = ctx.dictionary->lookup(s); + entry = ctx.dictionary.lookup(s); } if (!entry) { throw IntError{"`" + s_copy + "` not found"}; } else { - ctx.dictionary->undef_word(s); + ctx.dictionary.undef_word(s); } } // { bl word (forget) } : forget void interpret_forget(IntCtx& ctx) { - ctx.stack.push_string(ctx.scan_word()); + ctx.stack.push_string(ctx.parser->scan_word()); interpret_forget_aux(ctx); } void interpret_quote_str(IntCtx& ctx) { - ctx.stack.push_string(ctx.scan_word_to('"')); - push_argcount(ctx.stack, 1); + ctx.stack.push_string(ctx.parser->scan_word_to('"')); + push_argcount(ctx, 1); } int str_utf8_code(const char* str, int& len) { @@ -2056,7 +2241,7 @@ int str_utf8_code(const char* str, int& len) { } void interpret_char(IntCtx& ctx) { - auto s = ctx.scan_word(); + auto s = ctx.parser->scan_word(); int len = (s.size() < 10 ? (int)s.size() : 10); int code = str_utf8_code(s.data(), len); if (code < 0 || s.size() != (unsigned)len) { @@ -2197,6 +2382,12 @@ Ref interpret_execute(IntCtx& ctx) { return pop_exec_token(ctx); } +Ref interpret_call_cc(IntCtx& ctx) { + auto next = pop_exec_token(ctx); + ctx.stack.push_object(std::move(ctx.next)); + return next; +} + Ref interpret_execute_times(IntCtx& ctx) { int count = ctx.stack.pop_smallint_range(1000000000); auto body = pop_exec_token(ctx); @@ -2251,30 +2442,46 @@ Ref interpret_until(IntCtx& ctx) { return body; } -void interpret_tick(IntCtx& ctx) { - std::string word = ctx.scan_word().str(); - auto entry = ctx.dictionary->lookup(word); - if (!entry) { - entry = ctx.dictionary->lookup(word + ' '); +DictEntry context_lookup(IntCtx& ctx, std::string word, bool append_space = true) { + if (append_space) { + auto entry = context_lookup(ctx, word, false); if (!entry) { - throw IntError{"word `" + word + "` undefined"}; + entry = context_lookup(ctx, word + ' ', false); } + return entry; } - ctx.stack.push({vm::from_object, entry->get_def()}); + auto entry = ctx.context.lookup(word); + if (!entry && ctx.context != ctx.dictionary) { + entry = ctx.dictionary.lookup(word); + } + if (!entry && ctx.main_dictionary != ctx.context && ctx.main_dictionary != ctx.dictionary) { + entry = ctx.main_dictionary.lookup(word); + } + return entry; +} + +void interpret_tick(IntCtx& ctx) { + std::string word = ctx.parser->scan_word().str(); + auto entry = context_lookup(ctx, word); + if (!entry) { + throw IntError{"word `" + word + "` undefined"}; + } + ctx.stack.push_object(entry.get_def()); push_argcount(ctx, 1); } -void interpret_find(IntCtx& ctx) { +void interpret_find(IntCtx& ctx, int mode) { std::string word = ctx.stack.pop_string(); - auto entry = ctx.dictionary->lookup(word); - if (!entry) { - entry = ctx.dictionary->lookup(word + ' '); - } + auto entry = context_lookup(ctx, word, !(mode & 2)); if (!entry) { ctx.stack.push_bool(false); } else { - ctx.stack.push({vm::from_object, entry->get_def()}); - ctx.stack.push_bool(true); + ctx.stack.push_object(entry.get_def()); + if (!(mode & 1) || !entry.is_active()) { + ctx.stack.push_bool(true); + } else { + ctx.stack.push_smallint(1); + } } } @@ -2284,9 +2491,13 @@ void interpret_leave_source(IntCtx& ctx) { } } +void interpret_include_depth(IntCtx& ctx) { + ctx.stack.push_smallint(ctx.include_depth()); +} + Ref interpret_include(IntCtx& ctx) { auto fname = ctx.stack.pop_string(); - auto r_file = ctx.source_lookup->lookup_source(fname, ctx.currentd_dir); + auto r_file = ctx.source_lookup->lookup_source(fname, ctx.parser->currentd_dir); if (r_file.is_error()) { throw IntError{"cannot locate file `" + fname + "`"}; } @@ -2314,12 +2525,32 @@ Ref interpret_skip_source(IntCtx& ctx) { } void interpret_words(IntCtx& ctx) { - for (const auto& x : *ctx.dictionary) { - *ctx.output_stream << x.first << " "; + for (const auto& x : ctx.dictionary) { + *ctx.output_stream << vm::StackEntry(x.key()).as_string() << " "; } *ctx.output_stream << std::endl; } +void interpret_get_current(IntCtx& ctx) { + ctx.stack.push(ctx.dictionary.get_box()); +} + +void interpret_set_current(IntCtx& ctx) { + ctx.dictionary = ctx.stack.pop_box(); +} + +void interpret_get_context(IntCtx& ctx) { + ctx.stack.push(ctx.context.get_box()); +} + +void interpret_set_context(IntCtx& ctx) { + ctx.context = ctx.stack.pop_box(); +} + +void interpret_set_context_to(IntCtx& ctx, Ref box) { + ctx.context = std::move(box); +} + void interpret_print_backtrace(IntCtx& ctx) { ctx.print_backtrace(*ctx.output_stream, ctx.next); } @@ -2465,6 +2696,28 @@ void interpret_run_vm(IntCtx& ctx, int mode) { } } +void interpret_vmop_len(vm::Stack& stack) { + int cp = stack.pop_smallint_range(0x7fffffff, -0x80000000); + auto cs = stack.pop_cellslice(); + auto dispatch = vm::DispatchTable::get_table(cp); + if (!dispatch) { + throw IntError{"unknown vm codepage"}; + } + stack.push_smallint(dispatch->instr_len(*cs)); +} + +void interpret_vmop_dump(vm::Stack& stack) { + int cp = stack.pop_smallint_range(0x7fffffff, -0x80000000); + auto cs = stack.pop_cellslice(); + auto dispatch = vm::DispatchTable::get_table(cp); + if (!dispatch) { + throw IntError{"unknown vm codepage"}; + } + auto dump = dispatch->dump_instr(cs.write()); + stack.push_cellslice(std::move(cs)); + stack.push_string(std::move(dump)); +} + void do_interpret_db_run_vm_parallel(std::ostream* stream, vm::Stack& stack, vm::TonDb* ton_db_ptr, int threads_n, int tasks_n) { if (!ton_db_ptr || !*ton_db_ptr) { @@ -2623,11 +2876,11 @@ Ref interpret_get_cmdline_arg(IntCtx& ctx) { interpret_get_fixed_cmdline_arg(ctx.stack, n); return {}; } - auto entry = ctx.dictionary->lookup("$0 "); + auto entry = ctx.dictionary.lookup("$0 "); if (!entry) { throw IntError{"-?"}; } else { - return entry->get_def(); + return entry.get_def(); } } @@ -2667,29 +2920,19 @@ Ref interpret_execute_internal(IntCtx& ctx) { return word_def; } -// wl x1 .. xn n 'w --> wl' -void interpret_compile_internal(vm::Stack& stack) { - Ref word_def = pop_exec_token(stack); - int count = stack.pop_smallint_range(255); - do_compile_literals(stack, count); - if (word_def != Dictionary::nop_word_def) { - do_compile(stack, word_def); - } -} - void do_compile(vm::Stack& stack, Ref word_def) { Ref wl_ref = pop_word_list(stack); - if (word_def != Dictionary::nop_word_def) { + if (word_def != nop_word_def) { auto list_size = word_def->list_size(); - if ((td::uint64)list_size <= 1) { - // inline short definitions + if (list_size >= 0 && (list_size <= 2 || word_def.is_unique())) { + // inline short and unique definitions auto list = word_def->get_list(); wl_ref.write().append(list, list + list_size); } else { - wl_ref.write().push_back(word_def); + wl_ref.write().push_back(std::move(word_def)); } } - stack.push({vm::from_object, wl_ref}); + stack.push_object(std::move(wl_ref)); } void compile_one_literal(WordList& wlist, vm::StackEntry val) { @@ -2717,12 +2960,146 @@ void do_compile_literals(vm::Stack& stack, int count) { } } stack.pop_many(count + 1); - stack.push({vm::from_object, wl_ref}); + stack.push_object(std::move(wl_ref)); +} + +// wl x1 .. xn n 'w --> wl' +void interpret_compile_internal(vm::Stack& stack) { + Ref word_def = pop_exec_token(stack); + int count = stack.pop_smallint_range(255); + do_compile_literals(stack, count); + if (word_def != nop_word_def) { + do_compile(stack, std::move(word_def)); + } +} + +Ref interpret_compile_execute(IntCtx& ctx) { + if (ctx.state > 0) { + interpret_compile_internal(ctx.stack); + return {}; + } else { + return interpret_execute_internal(ctx); + } +} + +void interpret_seekeof(IntCtx& ctx, int mode) { + if (mode == -1) { + mode = ctx.stack.pop_smallint_range(3, -1); + } + bool eof = true; + if (ctx.parser && (ctx.parser->get_input() || ctx.parser->load_next_line())) { + while (true) { + if (!ctx.parser->is_sb()) { + ctx.parser->skipspc(); + if (*ctx.parser->get_input()) { + eof = false; + break; + } + } + if (mode & 1) { + *ctx.output_stream << " ok" << std::endl; + } + if (!ctx.parser->load_next_line()) { + break; + } + } + } + ctx.stack.push_bool(eof); +} + +void interpret_word_prefix_find(IntCtx& ctx, int mode) { + const char *ptr = ctx.parser->get_input(), *start = ptr; + if (!ptr) { + ctx.stack.push_string(std::string{}); + ctx.stack.push_bool(false); + return; + } + while (*ptr && *ptr != ' ' && *ptr != '\t') { + ptr++; + } + std::string Word{start, ptr}; + Word.push_back(' '); + auto entry = context_lookup(ctx, Word, false); + Word.pop_back(); + if (entry) { + ctx.parser->set_input(ptr); + ctx.parser->skipspc(); + } else { + const char* ptr_end = ptr; + while (true) { + entry = context_lookup(ctx, Word, false); + if (entry) { + ctx.parser->set_input(ptr); + break; + } + if (ptr == start) { + Word = std::string{start, ptr_end}; + ctx.parser->set_input(ptr_end); + ctx.parser->skipspc(); + break; + } + Word.pop_back(); + --ptr; + } + } + ctx.parser->word = Word; + if (!(mode & 2) || !entry) { + ctx.stack.push_string(std::move(Word)); + } + if (mode & 1) { + if (!entry) { + ctx.stack.push_bool(false); + } else { + ctx.stack.push_object(entry.get_def()); + ctx.stack.push_smallint(entry.is_active() ? 1 : -1); + } + } +} + +// equivalent to +// { ?dup { 1+ { execute } { 0 swap } cond } { (number) ?dup 0= abort"-?" 'nop } cond +// } : (interpret-prepare) +Ref interpret_prepare(IntCtx& ctx) { + int found = ctx.stack.pop_smallint_range(1, -1); + if (!found) { + // numbers + interpret_parse_number(ctx); // (number) + interpret_cond_dup(ctx); // ?dup + auto res = ctx.stack.pop_int(); // 0= abort"-?" + if (res == 0) { + throw IntError{"-?"}; + } + ctx.stack.push_object(nop_word_def); // 'nop + return {}; + } else if (found == -1) { + // ordinary word + ctx.stack.push_smallint(0); // 0 + interpret_swap(ctx); // swap + return {}; + } else { + // active word + return pop_exec_token(ctx); // execute + } +} + +Ref InterpretCont::run_tail(IntCtx& ctx) const { + static Ref interpret_prepare_ref = td::make_ref(interpret_prepare); + static Ref compile_exec_ref = td::make_ref(interpret_compile_execute); + interpret_seekeof(ctx, !ctx.state && !ctx.include_depth()); // seekeof + if (ctx.stack.pop_bool()) { + exit_interpret->clear(); + return {}; // exit loop + } + exit_interpret->set({vm::from_object, ctx.next}); // set 'exit-interpret to current continuation + interpret_word_prefix_find(ctx, 3); // (word-prefix-find) + // (interpet-prepare) (compile-execute) and schedule next loop iteration + ctx.next = SeqCont::seq(compile_exec_ref, SeqCont::seq(self(), std::move(ctx.next))); + return interpret_prepare_ref; // (interpret-prepare) } void init_words_common(Dictionary& d) { using namespace std::placeholders; - d.def_word("nop ", Dictionary::nop_word_def); + d.def_word("nop ", nop_word_def); // stack print/dump words d.def_ctx_word(". ", std::bind(interpret_dot, _1, true)); d.def_ctx_word("._ ", std::bind(interpret_dot, _1, false)); @@ -2942,6 +3319,14 @@ void init_words_common(Dictionary& d) { d.def_stack_word("ref@+ ", std::bind(interpret_fetch_ref, _1, 2)); d.def_stack_word("ref@? ", std::bind(interpret_fetch_ref, _1, 4)); d.def_stack_word("ref@?+ ", std::bind(interpret_fetch_ref, _1, 6)); + d.def_stack_word("s@ ", std::bind(interpret_fetch_slice, _1, 0)); + d.def_stack_word("sr@ ", std::bind(interpret_fetch_slice, _1, 1)); + d.def_stack_word("s@+ ", std::bind(interpret_fetch_slice, _1, 2)); + d.def_stack_word("sr@+ ", std::bind(interpret_fetch_slice, _1, 3)); + d.def_stack_word("s@? ", std::bind(interpret_fetch_slice, _1, 4)); + d.def_stack_word("sr@? ", std::bind(interpret_fetch_slice, _1, 5)); + d.def_stack_word("s@?+ ", std::bind(interpret_fetch_slice, _1, 6)); + d.def_stack_word("sr@?+ ", std::bind(interpret_fetch_slice, _1, 7)); d.def_stack_word("s> ", interpret_cell_check_empty); d.def_stack_word("empty? ", interpret_cell_empty); d.def_stack_word("remaining ", interpret_cell_remaining); @@ -2970,6 +3355,18 @@ void init_words_common(Dictionary& d) { d.def_stack_word("crc16 ", interpret_crc16); d.def_stack_word("crc32 ", interpret_crc32); d.def_stack_word("crc32c ", interpret_crc32c); + // hashmaps + d.def_stack_word("hmapnew ", interpret_hmap_new); + d.def_stack_word("hmap@ ", std::bind(interpret_hmap_fetch, _1, 6)); + d.def_stack_word("hmap@? ", std::bind(interpret_hmap_fetch, _1, 5)); + d.def_stack_word("hmap- ", std::bind(interpret_hmap_delete, _1, 0)); + d.def_stack_word("hmap-? ", std::bind(interpret_hmap_delete, _1, 1)); + d.def_stack_word("hmap@- ", std::bind(interpret_hmap_delete, _1, 6)); + d.def_stack_word("hmap! ", std::bind(interpret_hmap_store, _1, 0)); + d.def_stack_word("hmap!+ ", std::bind(interpret_hmap_store, _1, 1)); + d.def_stack_word("hmapempty? ", interpret_hmap_is_empty); + d.def_stack_word("hmapunpack ", std::bind(interpret_hmap_decompose, _1, 1)); + d.def_ctx_tail_word("hmapforeach ", std::bind(interpret_hmap_foreach, _1, 0)); // vm dictionaries d.def_stack_word("dictnew ", interpret_dict_new); d.def_stack_word("dict>s ", interpret_dict_to_slice); @@ -3041,6 +3438,7 @@ void init_words_common(Dictionary& d) { d.def_stack_word("atom? ", interpret_is_atom); // execution control d.def_ctx_tail_word("execute ", interpret_execute); + d.def_ctx_tail_word("call/cc ", interpret_call_cc); d.def_ctx_tail_word("times ", interpret_execute_times); d.def_ctx_tail_word("if ", interpret_if); d.def_ctx_tail_word("ifnot ", interpret_ifnot); @@ -3056,10 +3454,12 @@ void init_words_common(Dictionary& d) { d.def_stack_word("(}) ", interpret_wordlist_end_aux); d.def_stack_word("(compile) ", interpret_compile_internal); d.def_ctx_tail_word("(execute) ", interpret_execute_internal); + d.def_ctx_tail_word("(interpret-prepare) ", interpret_prepare); d.def_active_word("' ", interpret_tick); - d.def_word("'nop ", LitCont::literal({vm::from_object, Dictionary::nop_word_def})); + d.def_word("'nop ", LitCont::literal({vm::from_object, nop_word_def})); // dictionary manipulation - d.def_ctx_word("find ", interpret_find); + d.def_ctx_word("find ", std::bind(interpret_find, _1, 1)); + d.def_ctx_word("(word-prefix-find) ", std::bind(interpret_word_prefix_find, _1, 3)); d.def_ctx_word("create ", interpret_create); d.def_ctx_word("(create) ", std::bind(interpret_create_aux, _1, -1)); d.def_active_word(": ", std::bind(interpret_colon, _1, 0)); @@ -3069,12 +3469,21 @@ void init_words_common(Dictionary& d) { d.def_ctx_word("(forget) ", interpret_forget_aux); d.def_ctx_word("forget ", interpret_forget); d.def_ctx_word("words ", interpret_words); + d.def_word("Fift-wordlist ", LitCont::literal(d.get_box())); + d.def_ctx_word("Fift ", std::bind(interpret_set_context_to, _1, d.get_box())); + d.def_ctx_word("current@ ", interpret_get_current); + d.def_ctx_word("current! ", interpret_set_current); + d.def_ctx_word("context@ ", interpret_get_context); + d.def_ctx_word("context! ", interpret_set_context); d.def_ctx_word(".bt ", interpret_print_backtrace); d.def_ctx_word("cont. ", interpret_print_continuation); // input parse d.def_ctx_word("word ", interpret_word); d.def_ctx_word("(word) ", interpret_word_ext); d.def_ctx_word("skipspc ", interpret_skipspc); + d.def_ctx_word("seekeof? ", std::bind(interpret_seekeof, _1, 1)); + d.def_ctx_word("(seekeof?) ", std::bind(interpret_seekeof, _1, -1)); + d.def_ctx_word("include-depth ", interpret_include_depth); d.def_ctx_tail_word("include ", interpret_include); d.def_ctx_tail_word("skip-to-eof ", interpret_skip_source); d.def_word("'exit-interpret ", LitCont::literal(exit_interpret)); @@ -3110,6 +3519,8 @@ void init_words_vm(Dictionary& d, bool enable_debug) { d.def_ctx_word("dbrunvm-parallel ", interpret_db_run_vm_parallel); d.def_stack_word("vmcont, ", interpret_store_vm_cont); d.def_stack_word("vmcont@ ", interpret_fetch_vm_cont); + d.def_stack_word("(vmoplen) ", interpret_vmop_len); + d.def_stack_word("(vmopdump) ", interpret_vmop_dump); } void import_cmdline_args(Dictionary& d, std::string arg0, int n, const char* const argv[]) { @@ -3128,99 +3539,4 @@ void import_cmdline_args(Dictionary& d, std::string arg0, int n, const char* con } } -std::pair numeric_value_ext(std::string s, bool allow_frac = true) { - td::RefInt256 num, denom; - int res = parse_number(s, num, denom, allow_frac); - if (res <= 0) { - throw IntError{"-?"}; - } - return std::make_pair(std::move(num), res == 2 ? std::move(denom) : td::RefInt256{}); -} - -td::RefInt256 numeric_value(std::string s) { - td::RefInt256 num, denom; - int res = parse_number(s, num, denom, false); - if (res != 1) { - throw IntError{"-?"}; - } - return num; -} - -Ref interpret_compile_execute(IntCtx& ctx) { - if (ctx.state > 0) { - interpret_compile_internal(ctx.stack); - return {}; - } else { - return interpret_execute_internal(ctx); - } -} - -Ref InterpretCont::run_tail(IntCtx& ctx) const { - if (!ctx.get_input() && !ctx.load_next_line()) { - return {}; - } - while (true) { - if (!ctx.is_sb()) { - ctx.skipspc(); - if (*ctx.get_input()) { - break; - } - } - if (!ctx.state && !ctx.include_depth) { - *ctx.output_stream << " ok" << std::endl; - } - if (!ctx.load_next_line()) { - return {}; - } - } - const char* ptr = ctx.get_input(); - std::string Word; - Word.reserve(128); - auto entry = ctx.dictionary->lookup(""); - std::string entry_word; - const char* ptr_end = ptr; - while (*ptr && *ptr != ' ' && *ptr != '\t') { - Word += *ptr++; - auto cur = ctx.dictionary->lookup(Word); - if (cur) { - entry = cur; - entry_word = Word; - ptr_end = ptr; - } - } - auto cur = ctx.dictionary->lookup(Word + " "); - if (cur || !entry) { - entry = std::move(cur); - ctx.set_input(ptr); - ctx.skipspc(); - } else { - Word = entry_word; - ctx.set_input(ptr_end); - } - ctx.word = Word; - static Ref compile_exec_ref = td::make_ref(interpret_compile_execute); - if (!entry) { - // numbers - auto res = numeric_value_ext(Word); - ctx.stack.push(std::move(res.first)); - if (res.second.not_null()) { - ctx.stack.push(std::move(res.second)); - push_argcount(ctx, 2); - } else { - push_argcount(ctx, 1); - } - } else if (!entry->is_active()) { - // ordinary word - ctx.stack.push_smallint(0); - ctx.stack.push({vm::from_object, entry->get_def()}); - } else { - // active word - ctx.next = SeqCont::seq(compile_exec_ref, SeqCont::seq(self(), std::move(ctx.next))); - return entry->get_def(); - } - exit_interpret->set({vm::from_object, ctx.next}); - ctx.next = SeqCont::seq(self(), std::move(ctx.next)); - return compile_exec_ref; -} - } // namespace fift diff --git a/crypto/func/abscode.cpp b/crypto/func/abscode.cpp index 21d172cf..933edc3d 100644 --- a/crypto/func/abscode.cpp +++ b/crypto/func/abscode.cpp @@ -179,7 +179,7 @@ void VarDescr::set_const_nan() { void VarDescr::operator|=(const VarDescr& y) { val &= y.val; - if (is_int_const() && cmp(int_const, y.int_const) != 0) { + if (is_int_const() && y.is_int_const() && cmp(int_const, y.int_const) != 0) { val &= ~_Const; } if (!(val & _Const)) { diff --git a/crypto/func/analyzer.cpp b/crypto/func/analyzer.cpp index 5d9b3991..989e9ebe 100644 --- a/crypto/func/analyzer.cpp +++ b/crypto/func/analyzer.cpp @@ -388,7 +388,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { for (; l_it < left.cend(); ++l_it, ++r_it) { if (std::find(l_it + 1, left.cend(), *l_it) == left.cend()) { auto p = next_var_info[*l_it]; - new_var_info.add_var(*r_it, !p || p->is_unused()); + new_var_info.add_var(*r_it, edit && (!p || p->is_unused())); new_left.push_back(*l_it); new_right.push_back(*r_it); } @@ -500,7 +500,12 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { } changes = (new_var_info.size() == n); } while (changes <= edit); + assert(left.size() == 1); + bool last = new_var_info.count_used(left) == 0; new_var_info += left; + if (last) { + new_var_info[left[0]]->flags |= VarDescr::_Last; + } return set_var_info(std::move(new_var_info)); } case _Again: { diff --git a/crypto/func/auto-tests/run_tests.py b/crypto/func/auto-tests/run_tests.py index ae9c990c..0f12332d 100644 --- a/crypto/func/auto-tests/run_tests.py +++ b/crypto/func/auto-tests/run_tests.py @@ -66,6 +66,21 @@ for ti, tf in enumerate(tests): print("Error: no test cases", file=sys.stderr) exit(2) + # preprocess arithmetics in input + for i in range(len(cases)): + inputs = cases[i][1].split(" ") + processed_inputs = "" + for in_arg in inputs: + if "x{" in in_arg: + processed_inputs += in_arg + continue + # filter and execute + # is it safe enough? + filtered_in = "".join(filter(lambda x: x in "0x123456789()+-*/<>", in_arg)) + if(filtered_in): + processed_inputs += str(eval(filtered_in)) + " "; + cases[i][1] = processed_inputs.strip() + with open(RUNNER_FIF, "w") as f: print("\"%s\" include (tuple, ()) ~tset(tuple t, int idx, X val) asm(t val idx) "SETINDEXVAR"; + +;; computes 1-acos(x)/Pi by a very simple, extremely slow (~70k gas) and imprecise method +;; fixed256 acos_prepare_slow(fixed255 x); +int acos_prepare_slow_f255(int x) inline { + x -= (x == 0); + int t = 1; + repeat (255) { + t = t * sgn(x) * 2 + 1; ;; decode Gray code (sgn(x_0), sgn(x_1), ...) + x = (-1 << 255) - muldivr(x, - x, 1 << 254); ;; iterate x := 2*x^2 - 1 = cos(2*acos(x)) + } + return abs(t); +} + +;; extremely slow (~70k gas) and somewhat imprecise (very imprecise when x is small), for testing only +;; fixed254 acos_slow(fixed255 x); +int acos_slow_f255(int x) inline_ref { + int t = acos_prepare_slow_f255(x); + return - mulrshiftr256(t + (-1 << 256), Pi_const_f254()); +} + +;; fixed255 asin_slow(fixed255 x); +int asin_slow_f255(int x) inline_ref { + int t = acos_prepare_slow_f255(abs(x)) % (1 << 255); + return muldivr(t, Pi_const_f254(), 1 << 255) * sgn(x); +} + +tuple test_nrand(int n) inline_ref { + tuple t = empty_tuple(); + repeat (255) { + t~tpush(0); + } + repeat (n) { + int x = fixed248::nrand(); + int bucket = (abs(x) >> 243); ;; 255 buckets starting from x=0, each 1/32 wide + t~tset(bucket, t.at(bucket) + 1); + } + return t; +} + +int geom_mean_test(int x, int y) method_id(10000) { + return geom_mean(x, y); +} +int tan_f260_test(int x) method_id(10001) { + return tan_f260(x); +} +(int, int) sincosm1_f259_test(int x) method_id(10002) { + return sincosm1_f259(x); +} +(int, int) sincosn_f256_test(int x, int y) method_id(10003) { + return sincosn_f256(x, y); +} +(int, int) sincosm1_f256_test(int x) method_id(10004) { + return sincosm1_f256(x); +} +(int, int) tan_aux_f256_test(x) method_id(10005) { + return tan_aux_f256(x); +} +(int) fixed248::tan_test(x) method_id(10006) { + return fixed248::tan(x); +} +{- + (int) atanh_alt_f258_test(x) method_id(10007) { + return atanh_alt_f258(x); + } +-} +(int) atanh_f258_test(x, y) method_id(10008) { + return atanh_f258(x, y); +} +(int) atanh_f261_test(x, y) method_id(10009) { + return atanh_f261(x, y); +} + +(int, int) log2_aux_f256_test(x) method_id(10010) { + return log2_aux_f256(x); +} +(int, int) log_aux_f256_test(x) method_id(10011) { + return log_aux_f256(x); +} +int fixed248::pow_test(x, y) method_id(10012) { + return fixed248::pow(x, y); +} +int exp_log_div(x, y) method_id(10013) { + return fixed248::exp(fixed248::log(x << 248) ~/ y); +} +int fixed248::log_test(x) method_id(10014) { + return fixed248::log(x); +} +(int,int) log_aux_f257_test(x) method_id(10015) { + return log_aux_f257(x); +} +(int,int) fixed248::sincos_test(x) method_id(10016) { + return fixed248::sincos(x); +} +int fixed248::exp_test(x) method_id(10017) { + return fixed248::exp(x); +} +int fixed248::exp2_test(x) method_id(10018) { + return fixed248::exp2(x); +} +int expm1_f257_test(x) method_id(10019) { + return expm1_f257(x); +} +int atan_f255_test(x) method_id(10020) { + return atan_f255(x); +} +int atan_f259_test(x, n) method_id(10021) { + return atan_f259(x, n); +} +(int, int) atan_aux_f256_test(x) method_id(10022) { + return atan_aux_f256(x); +} +int asin_f255_test(x) method_id(10023) { + return asin_f255(x); +} +int asin_slow_f255_test(x) method_id(10024) { + return asin_slow_f255(x); +} +int acos_f255_test(x) method_id(10025) { + return acos_f255(x); +} +int acos_slow_f255_test(x) method_id(10026) { + return acos_slow_f255(x); +} +int fixed248::atan_test(x) method_id(10027) { + return fixed248::atan(x); +} +int fixed248::acot_test(x) method_id(10028) { + return fixed248::acot(x); +} + +_ main() { + int One = 1; + ;; repeat(76 / 4) { One *= 10000; } + int sqrt2 = geom_mean(One, 2 * One); + int sqrt3 = geom_mean(One, 3 * One); + ;; return geom_mean(-1 - (-1 << 256), -1 - (-1 << 256)); + ;; return geom_mean(-1 - (-1 << 256), -2 - (-1 << 256)); + ;; return geom_mean(-1 - (-1 << 256), 1 << 255); + ;; return (sqrt2, geom_mean(sqrt2, One)); ;; (sqrt(2), 2^(1/4)) + ;; return (sqrt3, geom_mean(sqrt3, One)); ;; (sqrt(3), 3^(1/4)) + ;; return geom_mean(3 << 254, 1 << 254); + ;; return geom_mean(3, 5); + ;; return tan_f260(115641670674223639132965820642403718536242645001775371762318060545014644837101 - 1); + ;; return tan_f260(15 << 252); ;; tan(15/256) * 2^260 + ;; return sincosm1_f259(1 << 255); ;; (sin,1-cos)(1/16) * 2^259 + ;; return sincosm1_f259(115641670674223639132965820642403718536242645001775371762318060545014644837101 - 1); + ;; return sincosm1_f256((1 << 255) - 1 + (1 << 255)); ;; (sin,1-cos)(1-2^(-256)) + ;; return sincosm1_f256(Pi_const_f254()); ;; (sin,1-cos)(Pi/4) + ;; return sincosn_f256(Pi_const_f254(), 0); ;; (sin,-cos)(Pi/4) + ;; return sincosn_f256((1 << 255) + 1, 0); ;; (sin,-cos)(1/2+1/2^256) + ;; return sincosn_f256(1 << 254, 0); + ;; return sincosn_f256(touch(15) << 252, 0); ;; (sin,-cos)(15/16) + ;; return sincosm1_f256(touch(15) << 252); ;; (sin,1-cos)(15/16) + ;; return sincosn_f256(60628596148627720713372490462954977108898896221398738326462025186323149077698, 0); ;; (sin,-cos)(Pi/6) + ;; return sincosm1_f256(60628596148627720713372490462954977108898896221398738326462025186323149077698); ;; (sin,1-cos)(Pi/6) + ;; return tan_aux_f256(1899 << 245); ;; (p,q) such that p/q=tan(1899/2048) + ;; return fixed248::tan(11 << 248); ;; tan(11) + ;; return atanh_alt_f258(1 << 252); ;; atanh(1/64) * 2^258 + ;; return atanh_f258(1 << 252, 18); ;; atanh(1/64) * 2^258 + ;; return atanh_f261(muldivr(64, 1 << 255, 55), 18); ;; atanh(1/55) * 2^261 + ;; return log2_aux_f256(1 << 255); + ;; return log2_aux_f256(-1 - (-1 << 256)); ;; log2(2-1/2^255))*2^256 ~ 2^256 - 1.43 + ;; return log_aux_f256(-1 - (-1 << 256)); + ;; return log_aux_f256(3); ;; log(3/2)*2^256 + ;; return fixed248::pow(3 << 248, 3 << 248); ;; 3^3 + ;; return fixed248::exp(fixed248::log(5 << 248) ~/ 7); ;; exp(log(5)/7) = 5^(1/7) + ;; return fixed248::log(Pi_const_f254() ~>> 6); ;; log(Pi) + ;; return atanh_alt_f258(1 << 255); ;; atanh(1/8) * 2^258 + ;; return atanh_f258(1 << 255, 37); ;; atanh(1/8) * 2^258 + ;; return atanh_f258(81877371507464127617551201542979628307507432471243237061821853600756754782485, 36); ;; atanh(sqrt(2)/8) * 2^258 + ;; return log_aux_f257(Pi_const_f254()); ;; log(Pi/4) + ;; return log_aux_f257(3 << 254); ;; log(3) + ;; return atanh_alt_f258(81877371507464127617551201542979628307507432471243237061821853600756754782485); ;; atanh(sqrt(2)/8) * 2^258 + ;; return fixed248::sincos(Pi_const_f254() ~/ (64 * 3)); ;; (sin,cos)(Pi/3) + ;; return fixed248::exp(3 << 248); ;; exp(3)*2^248 + ;; return fixed248::exp2((1 << 248) ~/ 5); ;; 2^(1/5)*2^248 + ;; return fixed248::pow(3 << 248, -3 << 247); ;; 3^(-1.5) + ;; return fixed248::pow(10 << 248, -70 << 248); ;; 10^(-70) + ;; return fixed248::pow(fixed248::Pi_const(), touch(3) << 248); ;; Pi^3 ~ 31.006, computed more precisely + ;; return fixed248::pow(fixed248::Pi_const(), fixed248::Pi_const()); ;; Pi^Pi, more precisely + ;; return fixed248::exp(fixed248::log(fixed248::Pi_const()) * 3); ;; Pi^3 ~ 31.006 + ;; return fixed248::exp(muldivr(fixed248::log(fixed248::Pi_const()), fixed248::Pi_const(), 1 << 248)); ;; Pi^Pi + ;; return fixed248::sin(fixed248::log(fixed248::exp(fixed248::Pi_const()))); ;; sin(log(e^Pi)) + ;; return expm1_f257(1 << 255); ;; (exp(1/4)-1)*2^256 + ;; return expm1_f257(-1 << 256); ;; (exp(-1/2)-1)*2^256 (argument out of range, will overflow) + ;; return expm1_f257(log2_const_f256()); ;; (exp(log(2)/2)-1)*2^256 + ;; return expm1_f257(- log2_const_f256()); ;; (exp(-log(2)/2)-1)*2^256 + ;; return tanh_f258(log2_const_f256(), 17); ;; tanh(log(2)/4)*2^258 + ;; return atan_f255(0xa0 << 247); + ;; return atan_f259(1 << 255, 26); ;; atan(1/16) + ;; return atan_f259(touch(2273) << 244, 26); ;; atan(2273/2^15) + ;; return atan_aux_f256(0xa0 << 248); + ;; return atan_aux_f256(-1 - (-1 << 256)); + ;; return atan_aux_f256(-1 << 256); + ;; return atan_aux_f256(1); ;; atan(1/2^256)*2^261 = 32 + ;;return fixed248::nrand(); + ;; return test_nrand(100000); + int One = touch(1 << 255); + ;; return asin_f255(One); + ;; return asin_f255(-2 * One ~/ -3); + int arg = muldivr(12, One, 17); ;; 12/17 + ;; return [ asin_slow_f255(arg), asin_f255(arg) ]; + ;; return [ acos_slow_f255(arg), acos_f255(arg) ]; + ;; return 4 * atan_f255(One ~/ 5) - atan_f255(One ~/ 239); ;; 4 * atan(1/5) - atan(1/239) = Pi/4 as fixed255 + int One = touch(1 << 248); + ;; return fixed248::atan(One) ~/ 5); ;; atan(1/5) + ;; return fixed248::acot(One ~/ 239); ;; atan(1/5) +} + +{- + method_id | in | out +TESTCASE | 10000 | -1-(-1<<256) -1-(-1<<256) | 115792089237316195423570985008687907853269984665640564039457584007913129639935 +TESTCASE | 10000 | -1-(-1<<256) -2-(-1<<256) | 115792089237316195423570985008687907853269984665640564039457584007913129639934 +TESTCASE | 10000 | -1-(-1<<256) 1<<255 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 +TESTCASE | 10000 | 1 2 | 1 +TESTCASE | 10000 | 1 3 | 2 +TESTCASE | 10000 | 3<<254, 1<<254 | 50139445418395255283694704271811692336355250894665672355503583528635147053497 +TESTCASE | 10000 | 3 5 | 4 +TESTCASE | 10001 | 115641670674223639132965820642403718536242645001775371762318060545014644837101-1 | 115792089237316195423570985008687907853269984665640564039457584007913129639935 +TESTCASE | 10001 | 15<<252 | 108679485937549714997960660780289583146059954551846264494610741505469565211201 + +TESTCASE | 10002 | 1<<255 | 57858359242454268843682786479537198006144860419130642837770554273561536355094 28938600351875109040123440645416448095273333920390487381363947585666516031269 +TESTCASE | 10002 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 | 90796875678616203090520439851979829600860326752181983760731669850687818036503 71369031536005973567205947792557760023823761636922618688720973932041901854510 +TESTCASE | 10002 | 115641670674223639132965820642403718536242645001775371762318060545014644837100 | 115341536360906404779899502576747487978354537254490211650198994186870666100480 115341536360906404779899502576747487978354537254490211650198994186870666100479 +TESTCASE | 10003 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 0 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 -81877371507464127617551201542979628307507432471243237061821853600756754782486 +TESTCASE | 10003 | (1<<255)+1 0 | 55513684748706254392157395574451324146997108788015526773113170656738693667657 -101617118319522600545601981648807607350213579319835970884288805016705398675944 +TESTCASE | 10003 | 1<<254 0 | 28647421327665059059430596260119789787021370826354543144805343654507971817712 -112192393597863122712065585177748900737784171216163716639418346853706594800924 +TESTCASE | 10003 | 15<<252 0 | 93337815620236900315136494926097782162348358704087992554326802765553037216157 -68526346066204767396483080633934170508153877799043171682610011603005473885083 +TESTCASE | 10004 | 15<<252 | 93337815620236900315136494926097782162348358704087992554326802765553037216158 94531486342222856054175808749507474690232213733194784713695144809815311509707 +TESTCASE | 10003 | 60628596148627720713372490462954977108898896221398738326462025186323149077698 0 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 -100278890836790510567389408543623384672710501789331344711007167057270294106993 +TESTCASE | 10004 | 60628596148627720713372490462954977108898896221398738326462025186323149077698 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 31026396801051369712363152930129046361118965752618438656900833901285671065886 +TESTCASE | 10005 | 1899<<245 | -115784979074977116522606932816046735344768048129666123117516779696532375620701 -86847621900007587791673148476644866514014227467564880140262768165345715058771 +TESTCASE | 10006 | 11<<248 | -102200470999497240398685962406597118965525125432278008915850368651878945159221 +TESTCASE | 10008 | 1<<252 18 | 7237594612640731814076778712183932891481921212865048737772958953246047977071 +TESTCASE! | 10009 | 64*(1<<255)//55 18 | 67377367986958444187782963285047188951340314639925508148698906136973510008513 +TESTCASE | 10010 | 1<<255 | 0 255 +TESTCASE | 10011 | -1-(-1<<256) | 80260960185991308862233904206310070533990667611589946606122867505419956976171 255 +TESTCASE | 10012 | 3<<248 3<<248 | 12212446911748192486079752325135052781399568695204278238536542063334587891712 +TESTCASE | 10013 | 5 7 | 569235245303856216139605450142923208167703167128528666640203654338408315932 +TESTCASE | 10014 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 | 517776035526939558040896860590142614178014859368681705591403663865964112176 +TESTCASE | 10008 | 1<<255 37 | 58200445412255555045265806996802932280233368707362818578692888102488340124094 +TESTCASE | 10008 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 36 | 82746618329032515754939514227666784789465120373484337368014239356561508382845 +TESTCASE | 10015 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 | -55942510554172181731996424203087263676819062449594753161692794122306202470292 256 +TESTCASE | 10015 | 3<<254 | -66622616410625360568738677407433830899150908037353507097280251369610028875158 256 +TESTCASE | 10016 | 90942894222941581070058735694432465663348344332098107489693037779484723616546//(64*3) | 391714417331212931903864877123528846377775397614575565277371746317462086355 226156424291633194186662080095093570025917938800079226639565593765455331328 +TESTCASE | 10017 | 3<<248 | 9084946421051389814103830025729847734065792062362132089390904679466687950835 +TESTCASE | 10018 | (1<<248)//5 | 519571025111621076330285524602776985448579272766894385941850747946908706857 +TESTCASE | 10012 | 3<<248 -3<<247 | 87047648295825095978636639360784188083950088358794570061638165848324908079 +TESTCASE | 10012 | 10<<248 -70<<248 | 45231 +TESTCASE | 10012 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 3<<248 | 14024537329227316173680050897643053638073167245065581681188087336877135047241 +TESTCASE | 10012 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 1420982722233462204219667745225507275989817880189032929526453715304448806508 | 16492303277433924047657446877966346821161732581471802839855102123372676002295 +TESTCASE | 10019 | 1<<255 | 65775792789545756849501669218806308540691279864498696756901136302101823231959 +TESTCASE | 10019 | -1<<255 | -51226238931640701466578648374135745377468902266335737558089915608594425303282 + +TESTCASE | 10020 | 160<<247 | 32340690885082755723307749066376646841771751777398167772823878380310576779097 +TESTCASE | 10021 | 1<<255 26 | 57820835337111819566482910321201859268121322500887685881159030272507322418551 +TESTCASE | 10021 | 2273<<244 26 | 64153929153128256059565403901040178355488584937372975321150754259394300105908 +TESTCASE | 10022 | 160<<248 | 18 -13775317617017974742132028403521581424991093186766868001115299479309514610238 +TESTCASE | 10022 | -1-(-1<<256) | 25 16312150880916231694896252427912541090503675654570543195394548083530005073282 +TESTCASE | 10022 | -1<<256 | -25 -16312150880916231694896252427912541090503675654570543195394548083530005073298 +TESTCASE | 10022 | 1 | 0 32 + +TESTCASE | 10023 | 1<<255 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 +TESTCASE | 10023 | (1-(1<<255))//-3 | 19675212872822715586637341573564384553677006914302429002469838095945333339604 +TESTCASE | 10023 | 12*(1<<255)//17 | 45371280744427205854111943101074857545572584208710061167826656461897302968384 +TESTCASE | 10024 | 12*(1<<255)//17 | 45371280744427205854111943101074857545572584208710061167826656461897302968387 +TESTCASE | 10025 | 12*(1<<255)//17 | 22785806739257187607973396296678804058887880061694023160933190658793710324081 +TESTCASE | 10026 | 12*(1<<255)//17 | 22785806739257187607973396296678804058887880061694023160933190658793710324080 + +TESTCASE | 10027 | (1<<248)//5 | 89284547973388213553327350968415123522888028497458323165947767504203347189 +TESTCASE | 10028 | (1<<248)//239 | 708598849781543798951441405045469962900811296151941404481049216461523216127 +-} diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index b8524bb9..29691034 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -29,8 +29,12 @@ using namespace std::literals::string_literals; int glob_func_cnt, undef_func_cnt, glob_var_cnt, const_cnt; std::vector glob_func, glob_vars; +std::set prohibited_var_names; SymDef* predefine_builtin_func(std::string name, TypeExpr* func_type) { + if (name.back() == '_') { + prohibited_var_names.insert(name); + } sym_idx_t name_idx = sym::symbols.lookup(name, 1); if (sym::symbols.is_keyword(name_idx)) { std::cerr << "fatal: global function `" << name << "` already defined as a keyword" << std::endl; diff --git a/crypto/func/func-main.cpp b/crypto/func/func-main.cpp index 829d95e4..c8208eee 100644 --- a/crypto/func/func-main.cpp +++ b/crypto/func/func-main.cpp @@ -123,5 +123,7 @@ int main(int argc, char* const argv[]) { sources.push_back(std::string(argv[optind++])); } + funC::read_callback = funC::fs_read_callback; + return funC::func_proceed(sources, *outs, std::cerr); } diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index be44d2ea..87bf23e9 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -30,6 +30,8 @@ #include "parser/lexer.h" #include #include "git.h" +#include +#include "td/utils/port/path.h" namespace funC { @@ -39,6 +41,28 @@ bool interactive = false; GlobalPragma pragma_allow_post_modification{"allow-post-modification"}; GlobalPragma pragma_compute_asm_ltr{"compute-asm-ltr"}; std::string generated_from, boc_output_filename; +ReadCallback::Callback read_callback; + +td::Result fs_read_callback(ReadCallback::Kind kind, const char* query) { + switch (kind) { + case ReadCallback::Kind::ReadFile: { + std::ifstream ifs{query}; + if (ifs.fail()) { + auto msg = std::string{"cannot open source file `"} + query + "`"; + return td::Status::Error(msg); + } + std::stringstream ss; + ss << ifs.rdbuf(); + return ss.str(); + } + case ReadCallback::Kind::Realpath: { + return td::realpath(td::CSlice(query)); + } + default: { + return td::Status::Error("Unknown query kind"); + } + } +} /* * diff --git a/crypto/func/func.h b/crypto/func/func.h index 1d534dc3..10039ffa 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -19,6 +19,7 @@ #pragma once #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #include "parser/srcread.h" #include "parser/lexer.h" #include "parser/symtable.h" +#include "td/utils/Status.h" namespace funC { @@ -39,7 +41,7 @@ extern std::string generated_from; constexpr int optimize_depth = 20; -const std::string func_version{"0.4.2"}; +const std::string func_version{"0.4.3"}; enum Keyword { _Eof = -1, @@ -838,6 +840,7 @@ struct SymValConst : sym::SymValBase { extern int glob_func_cnt, undef_func_cnt, glob_var_cnt; extern std::vector glob_func, glob_vars; +extern std::set prohibited_var_names; /* * @@ -845,6 +848,35 @@ extern std::vector glob_func, glob_vars; * */ +class ReadCallback { +public: + /// Noncopyable. + ReadCallback(ReadCallback const&) = delete; + ReadCallback& operator=(ReadCallback const&) = delete; + + enum class Kind + { + ReadFile, + Realpath + }; + + static std::string kindString(Kind _kind) + { + switch (_kind) + { + case Kind::ReadFile: + return "source"; + case Kind::Realpath: + return "realpath"; + default: + throw ""; // todo ? + } + } + + /// File reading or generic query callback. + using Callback = std::function(ReadCallback::Kind, const char*)>; +}; + // defined in parse-func.cpp bool parse_source(std::istream* is, const src::FileDescr* fdescr); bool parse_source_file(const char* filename, src::Lexem lex = {}, bool is_main = false); @@ -1691,6 +1723,9 @@ void define_builtins(); extern int verbosity, indent, opt_level; extern bool stack_layout_comments, op_rewrite_comments, program_envelope, asm_preamble, interactive; extern std::string generated_from, boc_output_filename; +extern ReadCallback::Callback read_callback; + +td::Result fs_read_callback(ReadCallback::Kind kind, const char* query); class GlobalPragma { public: diff --git a/crypto/func/gen-abscode.cpp b/crypto/func/gen-abscode.cpp index a2b98878..6af8e5b8 100644 --- a/crypto/func/gen-abscode.cpp +++ b/crypto/func/gen-abscode.cpp @@ -205,6 +205,10 @@ int Expr::predefine_vars() { case _Var: if (!sym) { assert(val < 0 && here.defined()); + if (prohibited_var_names.count(sym::symbols.get_name(~val))) { + throw src::ParseError{ + here, PSTRING() << "symbol `" << sym::symbols.get_name(~val) << "` cannot be redefined as a variable"}; + } sym = sym::define_symbol(~val, false, here); // std::cerr << "predefining variable " << sym::symbols.get_name(~val) << std::endl; if (!sym) { diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 8f6f7afe..06c695bf 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -22,8 +22,6 @@ #include "openssl/digest.hpp" #include "block/block.h" #include "block-parse.h" -#include -#include "td/utils/port/path.h" namespace sym { @@ -174,6 +172,10 @@ FormalArg parse_formal_arg(Lexer& lex, int fa_idx) { lex.expect(_Ident, "formal parameter name"); } loc = lex.cur().loc; + if (prohibited_var_names.count(sym::symbols.get_name(lex.cur().val))) { + throw src::ParseError{ + loc, PSTRING() << "symbol `" << sym::symbols.get_name(lex.cur().val) << "` cannot be redefined as a variable"}; + } SymDef* new_sym_def = sym::define_symbol(lex.cur().val, true, loc); if (!new_sym_def) { lex.cur().error_at("cannot define symbol `", "`"); @@ -399,7 +401,7 @@ bool check_global_func(const Lexem& cur, sym_idx_t func_name = 0) { def = sym::define_global_symbol(func_name, 0, cur.loc); assert(def && "cannot define global function"); ++undef_func_cnt; - make_new_glob_func(def, TypeExpr::new_hole()); // was: ... ::new_func() + make_new_glob_func(def, TypeExpr::new_func()); // was: ... ::new_func() return true; } SymVal* val = dynamic_cast(def->value); @@ -1371,6 +1373,10 @@ std::vector parse_type_var_list(Lexer& lex) { throw src::ParseError{lex.cur().loc, "free type identifier expected"}; } auto loc = lex.cur().loc; + if (prohibited_var_names.count(sym::symbols.get_name(lex.cur().val))) { + throw src::ParseError{loc, PSTRING() << "symbol `" << sym::symbols.get_name(lex.cur().val) + << "` cannot be redefined as a variable"}; + } SymDef* new_sym_def = sym::define_symbol(lex.cur().val, true, loc); if (!new_sym_def || new_sym_def->value) { lex.cur().error_at("redefined type variable `", "`"); @@ -1757,7 +1763,7 @@ bool parse_source_file(const char* filename, src::Lexem lex, bool is_main) { } } - auto path_res = td::realpath(td::CSlice(filename)); + auto path_res = read_callback(ReadCallback::Kind::Realpath, filename); if (path_res.is_error()) { auto error = path_res.move_as_error(); lex.error(error.message().c_str()); @@ -1784,17 +1790,19 @@ bool parse_source_file(const char* filename, src::Lexem lex, bool is_main) { source_files[real_filename] = cur_source; cur_source->is_main = is_main; source_fdescr.push_back(cur_source); - std::ifstream ifs{filename}; - if (ifs.fail()) { - auto msg = std::string{"cannot open source file `"} + filename + "`"; + auto file_res = read_callback(ReadCallback::Kind::ReadFile, filename); + if (file_res.is_error()) { + auto msg = file_res.move_as_error().message().str(); if (lex.tp) { lex.error(msg); } else { throw src::Fatal{msg}; } } + auto file_str = file_res.move_as_ok(); + std::stringstream ss{file_str}; inclusion_locations.push(lex.loc); - bool res = parse_source(&ifs, cur_source); + bool res = parse_source(&ss, cur_source); inclusion_locations.pop(); return res; } diff --git a/crypto/funcfiftlib/funcfiftlib.cpp b/crypto/funcfiftlib/funcfiftlib.cpp index 6c8912bc..070c3e0d 100644 --- a/crypto/funcfiftlib/funcfiftlib.cpp +++ b/crypto/funcfiftlib/funcfiftlib.cpp @@ -30,6 +30,7 @@ #include "td/utils/JsonBuilder.h" #include "fift/utils.h" #include "td/utils/base64.h" +#include "td/utils/Status.h" #include #include @@ -99,6 +100,40 @@ td::Result compile_internal(char *config_json) { return result_json.string_builder().as_cslice().str(); } +/// Callback used to retrieve additional source files or data. +/// +/// @param _kind The kind of callback (a string). +/// @param _data The data for the callback (a string). +/// @param o_contents A pointer to the contents of the file, if found. Allocated via malloc(). +/// @param o_error A pointer to an error message, if there is one. Allocated via malloc(). +/// +/// The callback implementor must use malloc() to allocate storage for +/// contents or error. The callback implementor must use free() to free +/// said storage after func_compile returns. +/// +/// If the callback is not supported, *o_contents and *o_error must be set to NULL. +typedef void (*CStyleReadFileCallback)(char const* _kind, char const* _data, char** o_contents, char** o_error); + +funC::ReadCallback::Callback wrapReadCallback(CStyleReadFileCallback _readCallback) +{ + funC::ReadCallback::Callback readCallback; + if (_readCallback) { + readCallback = [=](funC::ReadCallback::Kind _kind, char const* _data) -> td::Result { + char* contents_c = nullptr; + char* error_c = nullptr; + _readCallback(funC::ReadCallback::kindString(_kind).data(), _data, &contents_c, &error_c); + if (!contents_c && !error_c) { + return td::Status::Error("Callback not supported"); + } + if (contents_c) { + return contents_c; + } + return td::Status::Error(std::string(error_c)); + }; + } + return readCallback; +} + extern "C" { const char* version() { @@ -111,7 +146,13 @@ const char* version() { return strdup(version_json.string_builder().as_cslice().c_str()); } -const char *func_compile(char *config_json) { +const char *func_compile(char *config_json, CStyleReadFileCallback callback) { + if (callback) { + funC::read_callback = wrapReadCallback(callback); + } else { + funC::read_callback = funC::fs_read_callback; + } + auto res = compile_internal(config_json); if (res.is_error()) { diff --git a/crypto/parser/symtable.cpp b/crypto/parser/symtable.cpp index a8843da9..d52d9648 100644 --- a/crypto/parser/symtable.cpp +++ b/crypto/parser/symtable.cpp @@ -32,8 +32,8 @@ int scope_level; SymTable<100003> symbols; -SymDef* sym_def[symbols.hprime]; -SymDef* global_sym_def[symbols.hprime]; +SymDef* sym_def[symbols.hprime + 1]; +SymDef* global_sym_def[symbols.hprime + 1]; std::vector> symbol_stack; std::vector scope_opened_at; diff --git a/crypto/parser/symtable.h b/crypto/parser/symtable.h index 9489b2bc..51d59dfa 100644 --- a/crypto/parser/symtable.h +++ b/crypto/parser/symtable.h @@ -161,8 +161,8 @@ struct SymDef { } }; -extern SymDef* sym_def[symbols.hprime]; -extern SymDef* global_sym_def[symbols.hprime]; +extern SymDef* sym_def[symbols.hprime + 1]; +extern SymDef* global_sym_def[symbols.hprime + 1]; extern std::vector> symbol_stack; extern std::vector scope_opened_at; diff --git a/crypto/smartcont/LICENSE.LGPL b/crypto/smartcont/LICENSE.LGPL new file mode 100644 index 00000000..b482fc4e --- /dev/null +++ b/crypto/smartcont/LICENSE.LGPL @@ -0,0 +1,481 @@ + GNU LIBRARY GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1991 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the library GPL. It is + numbered 2 because it goes with version 2 of the ordinary GPL.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Library General Public License, applies to some +specially designated Free Software Foundation software, and to any +other libraries whose authors decide to use it. You can use it for +your libraries, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if +you distribute copies of the library, or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link a program with the library, you must provide +complete object files to the recipients so that they can relink them +with the library, after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + Our method of protecting your rights has two steps: (1) copyright +the library, and (2) offer you this license which gives you legal +permission to copy, distribute and/or modify the library. + + Also, for each distributor's protection, we want to make certain +that everyone understands that there is no warranty for this free +library. If the library is modified by someone else and passed on, we +want its recipients to know that what they have is not the original +version, so that any problems introduced by others will not reflect on +the original authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that companies distributing free +software will individually obtain patent licenses, thus in effect +transforming the program into proprietary software. To prevent this, +we have made it clear that any patent must be licensed for everyone's +free use or not licensed at all. + + Most GNU software, including some libraries, is covered by the ordinary +GNU General Public License, which was designed for utility programs. This +license, the GNU Library General Public License, applies to certain +designated libraries. This license is quite different from the ordinary +one; be sure to read it in full, and don't assume that anything in it is +the same as in the ordinary license. + + The reason we have a separate public license for some libraries is that +they blur the distinction we usually make between modifying or adding to a +program and simply using it. Linking a program with a library, without +changing the library, is in some sense simply using the library, and is +analogous to running a utility program or application program. However, in +a textual and legal sense, the linked executable is a combined work, a +derivative of the original library, and the ordinary General Public License +treats it as such. + + Because of this blurred distinction, using the ordinary General +Public License for libraries did not effectively promote software +sharing, because most developers did not use the libraries. We +concluded that weaker conditions might promote sharing better. + + However, unrestricted linking of non-free programs would deprive the +users of those programs of all benefit from the free status of the +libraries themselves. This Library General Public License is intended to +permit developers of non-free programs to use free libraries, while +preserving your freedom as a user of such programs to change the free +libraries that are incorporated in them. (We have not seen how to achieve +this as regards changes in header files, but we have achieved it as regards +changes in the actual functions of the Library.) The hope is that this +will lead to faster development of free libraries. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, while the latter only +works together with the library. + + Note that it is possible for a library to be covered by the ordinary +General Public License rather than by this special one. + + GNU LIBRARY GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library which +contains a notice placed by the copyright holder or other authorized +party saying it may be distributed under the terms of this Library +General Public License (also called "this License"). Each licensee is +addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also compile or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + c) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + d) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the source code distributed need not include anything that is normally +distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Library General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/crypto/smartcont/config-code.fc b/crypto/smartcont/config-code.fc index a638144e..b3aa04c4 100644 --- a/crypto/smartcont/config-code.fc +++ b/crypto/smartcont/config-code.fc @@ -1,4 +1,6 @@ ;; Simple configuration smart contract +;; Currently deployed config-contract in mainnet can be found +;; on https://verifier.ton.org/Ef9VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVbxn () set_conf_param(int index, cell value) impure { var cs = get_data().begin_parse(); @@ -397,6 +399,7 @@ int register_voting_proposal(slice cs, int msg_value) impure inline_ref { hash = -0xcd506e6c; ;; cannot set mandatory parameter to null } } + ;; Note, in config contract currently deployed in mainnet, this limit is 256 if (param_val.cell_depth() >= 128) { hash = -0xc2616456; ;; bad value } @@ -601,7 +604,7 @@ _ unpack_proposal(slice pstatus) inline_ref { voters_list = cons(voter_id, voters_list); } } until (~ f); - ;; Note there is a bug in config contract currently deployed in testnet2: + ;; Note there is a bug in config contract currently deployed in mainnet: ;; wins and losses are messed up var (rounds_remaining, wins, losses) = (rest~load_uint(8), rest~load_uint(8), rest~load_uint(8)); rest.end_parse(); diff --git a/crypto/smartcont/mathlib.fc b/crypto/smartcont/mathlib.fc index cc6d1a46..f2dfd73f 100644 --- a/crypto/smartcont/mathlib.fc +++ b/crypto/smartcont/mathlib.fc @@ -3,7 +3,25 @@ - FunC fixed-point mathematical library - -} - + +{- + This file is part of TON FunC Standard Library. + + FunC Standard Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + FunC Standard Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + +-} + +#include "stdlib.fc"; +#pragma version >=0.4.2; + {---------------- HIGH-LEVEL FUNCTION DECLARATIONS -----------------} {- Most functions declared here work either with integers or with fixed-point numbers of type `fixed248`. @@ -63,6 +81,17 @@ int fixed248::atan(int x) inline_ref; ;; fixed248 acot(fixed248 x); int fixed248::acot(int x) inline_ref; +;; random number uniformly distributed in [0..1) +;; fixed248 random(); +int fixed248::random() impure inline; +;; random number with standard normal distribution (2100 gas on average) +;; fixed248 nrand(); +int fixed248::nrand() impure inline; +;; generates a random number approximately distributed according to the standard normal distribution (1200 gas) +;; (fails chi-squared test, but it is shorter and faster than fixed248::nrand()) +;; fixed248 nrand_fast(); +int fixed248::nrand_fast() impure inline; + -} ;; end (declarations) {-------------------- INTERMEDIATE FUNCTIONS -----------------------} @@ -125,30 +154,27 @@ int asin_f255(int x); ;; fixed254 acos(fixed255 x); int acos_f255(int x); +;; generates normally distributed pseudo-random number +;; fixed252 nrand(); +int nrand_f252(int x); + +;; a faster and shorter variant of nrand_f252() that fails chi-squared test +;; (should suffice for most purposes) +;; fixed252 nrand_fast(); +int nrand_fast_f252(int x); -} ;; end (declarations) {---------------- MISSING OPERATIONS AND BUILT-INS -----------------} int sgn(int x) asm "SGN"; -int abs(int x) asm "ABS"; -int min(int x, int y) asm "MIN"; ;; compute floor(log2(x))+1 int log2_floor_p1(int x) asm "UBITSIZE"; -;; FunC compiler emits MULDIVC instruction, but Asm.fif usually does not know it -;; add the following line to your Asm.fif if necessary -;; x{A986} @Defop MULDIVC - -;; the following instructions might have been emitted by muldivmod() and muldivmodr() builtins, but FunC does not have these -;; x{A9A5} @Defop MULRSHIFTR int mulrshiftr(int x, int y, int s) asm "MULRSHIFTR"; -;; x{A9B5} @Defop(8u+1) MULRSHIFTR# int mulrshiftr256(int x, int y) asm "256 MULRSHIFTR#"; -;; x{A9BC} @Defop(8u+1) MULRSHIFT#MOD (int, int) mulrshift256mod(int x, int y) asm "256 MULRSHIFT#MOD"; -;; x{A9BD} @Defop(8u+1) MULRSHIFTR#MOD (int, int) mulrshiftr256mod(int x, int y) asm "256 MULRSHIFTR#MOD"; (int, int) mulrshiftr255mod(int x, int y) asm "255 MULRSHIFTR#MOD"; (int, int) mulrshiftr248mod(int x, int y) asm "248 MULRSHIFTR#MOD"; @@ -156,39 +182,34 @@ int mulrshiftr256(int x, int y) asm "256 MULRSHIFTR#"; (int, int) mulrshiftr6mod(int x, int y) asm "6 MULRSHIFTR#MOD"; (int, int) mulrshiftr7mod(int x, int y) asm "7 MULRSHIFTR#MOD"; -;; these instructions might have been emitted by muldivmodr(..., 1 << N, ...) built-ins -;; x{A9D5} @Defop(8u+1) LSHIFT#DIVR int lshift256divr(int x, int y) asm "256 LSHIFT#DIVR"; -;; x{A9DD} @Defop(8u+1) LSHIFT#DIVMODR (int, int) lshift256divmodr(int x, int y) asm "256 LSHIFT#DIVMODR"; (int, int) lshift255divmodr(int x, int y) asm "255 LSHIFT#DIVMODR"; (int, int) lshift2divmodr(int x, int y) asm "2 LSHIFT#DIVMODR"; (int, int) lshift7divmodr(int x, int y) asm "7 LSHIFT#DIVMODR"; -;; x{A9CD} @Defop LSHIFTDIVMODR (int, int) lshiftdivmodr(int x, int y, int s) asm "LSHIFTDIVMODR"; -;; these instructions might have been emitted by divmodr(..., 1 << NN) or divmod(..., 1 << N) built-ins -;; but FunC does not have divmodr(), and divmod() always compiles to "DIVMOD" -;; x{A93D} @Defop(8u+1) RSHIFTR#MOD (int, int) rshiftr256mod(int x) asm "256 RSHIFTR#MOD"; (int, int) rshiftr248mod(int x) asm "248 RSHIFTR#MOD"; (int, int) rshiftr4mod(int x) asm "4 RSHIFTR#MOD"; -;; x{A93C} @Defop(8u+1) RSHIFT#MOD (int, int) rshift3mod(int x) asm "3 RSHIFT#MOD"; ;; computes y - x (FunC compiler does not try to use this by itself) int sub_rev(int x, int y) asm "SUBR"; +int nan() asm "PUSHNAN"; +int is_nan(int x) asm "ISNAN"; + {------------------------ SQUARE ROOTS ----------------------------} ;; computes sqrt(a*b) exactly rounded to the nearest integer ;; for all 0 <= a, b <= 2^256-1 ;; may be used with b=1 or b=scale of fixed-point numbers int geom_mean(int a, int b) inline_ref { - if (min(a, b) <= 0) { + ifnot (min(a, b)) { return 0; } - int s = log2_floor_p1(a); + int s = log2_floor_p1(a); ;; throws out of range error if a < 0 or b < 0 int t = log2_floor_p1(b); ;; NB: (a-b)/2+b == (a+b)/2, but without overflow for large a and b int x = (s == t ? (a - b) / 2 + b : 1 << ((s + t) / 2)); @@ -313,9 +334,6 @@ int expm1_f257(int x) inline_ref { return x - muldivr(x2, t / 2, a + mulrshiftr256(x, t) ~/ 4) ~/ 4; ;; x - x^2 * (x-a) / (a + x*(x-a)) } -;; used to avoid broken optimisations in older versions of FunC -int minus_one() asm "-1 PUSHINT"; - ;; expm1_f257() may be used to implement specific fixed-point exponentials ;; example: ;; fixed248 exp(fixed248 x) @@ -327,8 +345,7 @@ int fixed248::exp(int x) inline_ref { x = 2 * x - muldivr(q, l2d, 1 << 127); int y = expm1_f257(x); ;; result is (1 + y) * (2^q) --> ((1 << 257) + y) >> (9 - q) - return (y ~>> (9 - q)) - (minus_one() << (248 + q)); - ;; (-1 << (248 + q)) is compiled into non-existent instruction NEGPOW2 + return (y ~>> (9 - q)) - (-1 << (248 + q)); ;; note that (y ~>> (9 - q)) + (1 << (248 + q)) leads to overflow when q=8 } @@ -339,7 +356,7 @@ int fixed248::exp2(int x) inline_ref { (int q, x) = rshiftr248mod(x); x = muldivr(x, log2_const_f256(), 1 << 247); int y = expm1_f257(x); - return (y ~>> (9 - q)) - (minus_one() << (248 + q)); + return (y ~>> (9 - q)) - (-1 << (248 + q)); } {--------------------- TRIGONOMETRIC FUNCTIONS -----------------------} @@ -662,7 +679,7 @@ int fixed248::pow(int x, int y) inline_ref { return - (sq == 0); ;; underflow } int y = expm1_f257(mulrshiftr256(ll, log2_const_f256())); - return (y ~>> (9 - q)) - (minus_one() << sq); + return (y ~>> (9 - q)) - (-1 << sq); } {--------------------- INVERSE TRIGONOMETRIC FUNCTIONS -------------------} @@ -795,7 +812,7 @@ int atan_f256_small(int x) inline_ref { ;; fixed255 asin(fixed255 x); int asin_f255(int x) inline_ref { int a = fixed255::One - fixed255::sqr(x); ;; a:=1-x^2 - if (a <= 0) { + ifnot (a) { return sgn(x) * Pi_const_f254(); ;; Pi/2 or -Pi/2 } int y = fixed255::sqrt(a); ;; sqrt(1-x^2) @@ -806,7 +823,7 @@ int asin_f255(int x) inline_ref { ;; fixed254 acos(fixed255 x); int acos_f255(int x) inline_ref { int Pi = Pi_const_f254(); - if (x <= (-1 << 255)) { + if (x == (-1 << 255)) { return Pi; ;; acos(-1) = Pi } Pi /= 2; @@ -858,3 +875,65 @@ int fixed248::acot(int x) inline_ref { ;; now acot(x) = - z - q*atan(1/32) + s*(Pi/2), z is fixed261 return (s * Pi_const_f254() - z ~/ 64 - muldivr(q, Atan1_32_f261(), 64)) ~/ 128; ;; compute in fixed255, then convert } + +{--------------------- PSEUDO-RANDOM NUMBERS -------------------} + +;; random number with standard normal distribution N(0,1) +;; generated by Kinderman--Monahan ratio method modified by J.Leva +;; spends ~ 2k..3k gas on average +;; fixed252 nrand(); +int nrand_f252() impure inline_ref { + var (x, s, t, A, B, r0) = (nan(), touch(29483) << 236, touch(-3167) << 239, 12845, 16693, 9043); + ;; 4/sqrt(e*Pi) = 1.369 loop iterations on average + do { + var (u, v) = (random() / 16 + 1, muldivr(random() - (1 << 255), 7027, 1 << 16)); ;; fixed252; 7027=ceil(sqrt(8/e)*2^12) + int va = abs(v); + var (u1, v1) = (u - s, va - t); ;; (u - 29483/2^16, abs(v) + 3167/2^13) as fixed252 + ;; Q := u1^2 + v1 * (A*v1 - B*u1) as fixed252 where A=12845/2^16, B=16693/2^16 + int Q = muldivr(u1, u1, 1 << 252) + muldivr(v1, muldivr(v1, A, 1 << 16) - muldivr(u1, B, 1 << 16), 1 << 252); + ;; must have 9043 / 2^15 < Q < 9125 / 2^15, otherwise accept if smaller, reject if larger + int Qd = (Q >> 237) - r0; + if ((Qd < 9125 - 9043) & (va / u < 16)) { + x = muldivr(v, 1 << 252, u); ;; x:=v/u as fixed252; reject immediately if |v/u| >= 16 + if (Qd >= 0) { ;; immediately accept if Qd < 0 + ;; rarely taken branch - 0.012 times per call on average + ;; check condition v^2 < -4*u^2*log(u), or equivalent condition u < exp(-x^2/4) for x=v/u + int xx = mulrshiftr256(x, x) ~/ 4; ;; x^2/4 as fixed248 + int ex = fixed248::exp(- xx) * 16; ;; exp(-x^2/4) as fixed252 + if (u > ex) { + x = nan(); ;; condition false, reject + } + } + } + } until (~ is_nan(x)); + return x; +} + +;; generates a random number approximately distributed according to the standard normal distribution +;; much faster than nrand_f252(), should be suitable for most purposes when only several random numbers are needed +;; fixed252 nrand_fast(); +int nrand_fast_f252() impure inline_ref { + int t = touch(-3) << 253; ;; -6. as fixed252 + repeat (12) { + t += random() / 16; ;; add together 12 uniformly random numbers + } + return t; +} + +;; random number uniformly distributed in [0..1) +;; fixed248 random(); +int fixed248::random() impure inline { + return random() >> 8; +} + +;; random number with standard normal distribution +;; fixed248 nrand(); +int fixed248::nrand() impure inline { + return nrand_f252() ~>> 4; +} + +;; generates a random number approximately distributed according to the standard normal distribution +;; fixed248 nrand_fast(); +int fixed248::nrand_fast() impure inline { + return nrand_fast_f252() ~>> 4; +} diff --git a/crypto/smartcont/stdlib.fc b/crypto/smartcont/stdlib.fc index 03da3fe4..978b9473 100644 --- a/crypto/smartcont/stdlib.fc +++ b/crypto/smartcont/stdlib.fc @@ -1,6 +1,21 @@ ;; Standard library for funC ;; +{- + This file is part of TON FunC Standard Library. + + FunC Standard Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + FunC Standard Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + +-} + {- # Tuple manipulation primitives The names and the types are mostly self-explaining. diff --git a/crypto/smartcont/test-math.fc b/crypto/smartcont/test-math.fc deleted file mode 100644 index 1ff43cd9..00000000 --- a/crypto/smartcont/test-math.fc +++ /dev/null @@ -1,100 +0,0 @@ -{-------------------------- TESTS for mathlib.fc ----------------------------} - -;; computes 1-acos(x)/Pi by a very simple, extremely slow (~70k gas) and imprecise method -;; fixed256 acos_prepare_slow(fixed255 x); -int acos_prepare_slow_f255(int x) inline { - x -= (x == 0); - int t = 1; - repeat (255) { - t = t * sgn(x) * 2 + 1; ;; decode Gray code (sgn(x_0), sgn(x_1), ...) - x = (-1 << 255) - muldivr(x, - x, 1 << 254); ;; iterate x := 2*x^2 - 1 = cos(2*acos(x)) - } - return abs(t); -} - -;; extremely slow (~70k gas) and somewhat imprecise (very imprecise when x is small), for testing only -;; fixed254 acos_slow(fixed255 x); -int acos_slow_f255(int x) inline_ref { - int t = acos_prepare_slow_f255(x); - return - mulrshiftr256(t + (-1 << 256), Pi_const_f254()); -} - -;; fixed255 asin_slow(fixed255 x); -int asin_slow_f255(int x) inline_ref { - int t = acos_prepare_slow_f255(abs(x)) % (1 << 255); - return muldivr(t, Pi_const_f254(), 1 << 255) * sgn(x); -} - -_ main() { - int One = 1; - ;; repeat(76 / 4) { One *= 10000; } - int sqrt2 = geom_mean(One, 2 * One); - int sqrt3 = geom_mean(One, 3 * One); - ;; return geom_mean((1 << 255) - 1 + (1 << 255), (1 << 255) - 1); - ;; return geom_mean((1 << 255) - 1, (1 << 255) - 2); - ;; return (sqrt2, geom_mean(sqrt2, One)); ;; (sqrt(2), 2^(1/4)) - ;; return (sqrt3, geom_mean(sqrt3, One)); ;; (sqrt(3), 3^(1/4)) - ;; return geom_mean(3 << 254, 1 << 254); - ;; return tan_f260(115641670674223639132965820642403718536242645001775371762318060545014644837101 - 1); - ;; return tan_f260(15 << 252); ;; tan(15/256) * 2^260 - ;; return sincosm1_f259(1 << 255); ;; (sin,1-cos)(1/16) * 2^259 - ;; return sincosm1_f259(115641670674223639132965820642403718536242645001775371762318060545014644837101 - 1); - ;; return sincosm1_f256((1 << 255) - 1 + (1 << 255)); ;; (sin,1-cos)(1-2^(-256)) - ;; return sincosm1_f256(Pi_const_f254()); ;; (sin,1-cos)(Pi/4) - ;; return sincosn_f256(Pi_const_f254(), 0); ;; (sin,-cos)(Pi/4) - ;; return sincosn_f256((1 << 255) + 1, 0); ;; (sin,-cos)(1/2+1/2^256) - ;; return sincosn_f256(1 << 254, 0); - ;; return sincosn_f256(touch(15) << 252, 0); ;; (sin,-cos)(15/16) - ;; return sincosm1_f256(touch(15) << 252); ;; (sin,1-cos)(15/16) - ;; return sincosn_f256(60628596148627720713372490462954977108898896221398738326462025186323149077698, 0); ;; (sin,-cos)(Pi/6) - ;; return sincosm1_f256(60628596148627720713372490462954977108898896221398738326462025186323149077698); ;; (sin,1-cos)(Pi/6) - ;; return tan_aux_f256(1899 << 245); ;; (p,q) such that p/q=tan(1899/2048) - ;; return fixed248::tan(11 << 248); ;; tan(11) - ;; return atanh_alt_f258(1 << 252); ;; atanh(1/64) * 2^258 - ;; return atanh_f258(1 << 252, 18); ;; atanh(1/64) * 2^258 - ;; return atanh_f261(muldivr(64, 1 << 255, 55), 18); ;; atanh(1/55) * 2^261 - ;; return log2_aux_f256(1 << 255); - ;; return log2_aux_f256(-1 - (-1 << 256)); ;; log2(2-1/2^255))*2^256 ~ 2^256 - 1.43 - ;; return log_aux_f256(-1 - (-1 << 256)); - ;; return log_aux_f256(3); ;; log(3/2)*2^256 - ;; return fixed248::pow(3 << 248, 3 << 248); ;; 3^3 - ;; return fixed248::exp(fixed248::log(5 << 248) ~/ 7); ;; exp(log(5)/7) = 5^(1/7) - ;; return fixed248::log(Pi_const_f254() ~>> 6); ;; log(Pi) - ;; return atanh_alt_f258(1 << 255); ;; atanh(1/8) * 2^258 - ;; return atanh_f258(1 << 255, 37); ;; atanh(1/8) * 2^258 - ;; return log_aux_f257(Pi_const_f254()); ;; log(Pi/4) - ;; return log_aux_f257(3 << 254); ;; log(3) - ;; return atanh_alt_f258(81877371507464127617551201542979628307507432471243237061821853600756754782485); ;; atanh(sqrt(2)/8) * 2^258 - ;; return atanh_f258(81877371507464127617551201542979628307507432471243237061821853600756754782485, 36); ;; atanh(sqrt(2)/8) * 2^258 - ;; return fixed248::sincos(Pi_const_f254() ~/ (64 * 3)); ;; (sin,cos)(Pi/3) - ;; return fixed248::exp(3 << 248); ;; exp(3)*2^248 - ;; return fixed248::exp2((1 << 248) ~/ 5); ;; 2^(1/5)*2^248 - ;; return fixed248::pow(3 << 248, -3 << 247); ;; 3^(-1.5) - ;; return fixed248::pow(10 << 248, -70 << 248); ;; 10^(-70) - ;; return fixed248::exp(fixed248::log(fixed248::Pi_const()) * 3); ;; Pi^3 ~ 31.006 - ;; return fixed248::pow(fixed248::Pi_const(), touch(3) << 248); ;; Pi^3 ~ 31.006, computed more precisely - ;; return fixed248::exp(muldivr(fixed248::log(fixed248::Pi_const()), fixed248::Pi_const(), 1 << 248)); ;; Pi^Pi - ;; return fixed248::pow(fixed248::Pi_const(), fixed248::Pi_const()); ;; Pi^Pi, more precisely - ;; return fixed248::sin(fixed248::log(fixed248::exp(fixed248::Pi_const()))); ;; sin(log(e^Pi)) - ;; return expm1_f257(1 << 255); ;; (exp(1/4)-1)*2^256 - ;; return expm1_f257(-1 << 256); ;; (exp(-1/2)-1)*2^256 (argument out of range, will overflow) - ;; return expm1_f257(log2_const_f256()); ;; (exp(log(2)/2)-1)*2^256 - ;; return expm1_f257(- log2_const_f256()); ;; (exp(-log(2)/2)-1)*2^256 - ;; return tanh_f258(log2_const_f256(), 17); ;; tanh(log(2)/4)*2^258 - ;; return atan_f255(0xa0 << 247); - ;; return atan_f259(1 << 255, 26); ;; atan(1/16) - ;; return atan_f259(touch(2273) << 244, 26); ;; atan(2273/2^15) - ;; return atan_aux_f256(0xa0 << 248); - ;; return atan_aux_f256(-1 - (-1 << 256)); - ;; return atan_aux_f256(-1 << 256); - ;; return atan_aux_f256(1); ;; atan(1/2^256)*2^261 = 32 - int One = touch(1 << 255); - ;; return asin_f255(-2 * One ~/ -3); - int arg = muldivr(12, One, 17); ;; 12/17 - ;; return [ asin_slow_f255(arg), asin_f255(arg) ]; - ;; return [ acos_slow_f255(arg), acos_f255(arg) ]; - return 4 * atan_f255(One ~/ 5) - atan_f255(One ~/ 239); ;; 4 * atan(1/5) - atan(1/239) = Pi/4 as fixed255 - int One = touch(1 << 248); - ;; return fixed248::atan(One) ~/ 5); ;; atan(1/5) - ;; return fixed248::acot(One ~/ 239); ;; atan(1/5) -} diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 713caf9c..7ba768a4 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -40,14 +40,82 @@ int SmartContract::Answer::output_actions_count(td::Ref list) { } namespace { -td::Ref prepare_vm_stack(td::RefInt256 amount, td::Ref body) { +td::Ref build_internal_message(td::RefInt256 amount, td::Ref body, SmartContract::Args args) { + vm::CellBuilder cb; + if (args.address) { + td::BigInt256 dest_addr; + dest_addr.import_bits((*args.address).addr.as_bitslice()); + cb.store_ones(1) + .store_zeroes(2) + .store_long((*args.address).workchain, 8) + .store_int256(dest_addr, 256); + } + auto address = cb.finalize(); + + vm::CellBuilder b; + b.store_long(0b0110, 4); // 0 ihr_disabled:Bool bounce:Bool bounced:Bool + // use -1:00..00 as src:MsgAddressInt + // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + b.store_long(0b100, 3); b.store_ones(8); b.store_zeroes(256); + b.append_cellslice(address); // dest:MsgAddressInt + unsigned len = (((unsigned)amount->bit_size(false) + 7) >> 3); + b.store_long_bool(len, 4) && b.store_int256_bool(*amount, len * 8, false); // grams:Grams + b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extre, ihr_fee, fwd_fee, created_lt, created_at, init + // body:(Either X ^X) + if (b.remaining_bits() >= 1 + (*body).size() && b.remaining_refs() >= (*body).size_refs()) { + b.store_zeroes(1); + b.append_cellslice(body); + } else { + b.store_ones(1); + b.store_ref(vm::CellBuilder().append_cellslice(body).finalize_novm()); + } + return b.finalize_novm(); +} + +td::Ref build_external_message(td::RefInt256 amount, td::Ref body, SmartContract::Args args) { + vm::CellBuilder cb; + if (args.address) { + td::BigInt256 dest_addr; + dest_addr.import_bits((*args.address).addr.as_bitslice()); + cb.store_ones(1) + .store_zeroes(2) + .store_long((*args.address).workchain, 8) + .store_int256(dest_addr, 256); + } + auto address = cb.finalize(); + + vm::CellBuilder b; + b.store_long(0b1000, 4); // ext_in_msg_info$10 src:MsgAddressExt + b.append_cellslice(address); // dest:MsgAddressInt + b.store_zeroes(4); //import_fee:Grams + b.store_zeroes(1); // init + // body:(Either X ^X) + if (b.remaining_bits() >= 1 + (*body).size() && b.remaining_refs() >= (*body).size_refs()) { + b.store_zeroes(1); + b.append_cellslice(body); + } else { + b.store_ones(1); + b.store_ref(vm::CellBuilder().append_cellslice(body).finalize_novm()); + } + return b.finalize_novm(); +} + +td::Ref prepare_vm_stack(td::RefInt256 amount, td::Ref body, SmartContract::Args args, int selector) { td::Ref stack_ref{true}; td::RefInt256 acc_addr{true}; //CHECK(acc_addr.write().import_bits(account.addr.cbits(), 256)); vm::Stack& stack = stack_ref.write(); - stack.push_int(td::make_refint(10000000000)); - stack.push_int(std::move(amount)); - stack.push_cell(vm::CellBuilder().finalize()); + if(args.balance) { + stack.push_int(td::make_refint(args.balance)); + } else { + stack.push_int(td::make_refint(10000000000)); + } + stack.push_int(amount); + if(selector == 0) { + stack.push_cell(build_internal_message(amount, body, args)); + } else { + stack.push_cell(build_external_message(amount, body, args)); + } stack.push_cellslice(std::move(body)); return stack_ref; } @@ -100,9 +168,9 @@ td::Ref prepare_vm_c7(SmartContract::Args args) { } SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref stack, td::Ref c7, - vm::GasLimits gas, bool ignore_chksig, td::Ref libraries, int vm_log_verbosity) { + vm::GasLimits gas, bool ignore_chksig, td::Ref libraries, int vm_log_verbosity, bool debug_enabled) { auto gas_credit = gas.gas_credit; - vm::init_op_cp0(); + vm::init_op_cp0(debug_enabled); vm::DictionaryBase::get_empty_dictionary(); class Logger : public td::LogInterface { @@ -143,7 +211,7 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref= VERBOSITY_NAME(DEBUG)) { LOG(DEBUG) << "VM log\n" << logger.res; @@ -221,7 +289,7 @@ SmartContract::Answer SmartContract::run_method(Args args) { args.stack.value().write().push_smallint(args.method_id.unwrap()); auto res = run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig, - args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}, args.vm_log_verbosity_level); + args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}, args.vm_log_verbosity_level, args.debug_enabled); state_ = res.new_state; return res; } @@ -239,7 +307,7 @@ SmartContract::Answer SmartContract::run_get_method(Args args) const { CHECK(args.method_id); args.stack.value().write().push_smallint(args.method_id.unwrap()); return run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig, - args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}, args.vm_log_verbosity_level); + args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}, args.vm_log_verbosity_level, args.debug_enabled); } SmartContract::Answer SmartContract::run_get_method(td::Slice method, Args args) const { @@ -248,10 +316,10 @@ SmartContract::Answer SmartContract::run_get_method(td::Slice method, Args args) SmartContract::Answer SmartContract::send_external_message(td::Ref cell, Args args) { return run_method( - args.set_stack(prepare_vm_stack(td::make_refint(0), vm::load_cell_slice_ref(cell))).set_method_id(-1)); + args.set_stack(prepare_vm_stack(td::make_refint(0), vm::load_cell_slice_ref(cell), args, -1)).set_method_id(-1)); } SmartContract::Answer SmartContract::send_internal_message(td::Ref cell, Args args) { return run_method( - args.set_stack(prepare_vm_stack(td::make_refint(args.amount), vm::load_cell_slice_ref(cell))).set_method_id(0)); + args.set_stack(prepare_vm_stack(td::make_refint(args.amount), vm::load_cell_slice_ref(cell), args, 0)).set_method_id(0)); } } // namespace ton diff --git a/crypto/smc-envelope/SmartContract.h b/crypto/smc-envelope/SmartContract.h index 6da74bc6..ce349de3 100644 --- a/crypto/smc-envelope/SmartContract.h +++ b/crypto/smc-envelope/SmartContract.h @@ -65,6 +65,7 @@ class SmartContract : public td::CntObject { td::uint64 amount{0}; td::uint64 balance{0}; int vm_log_verbosity_level{0}; + bool debug_enabled{false}; td::optional address; td::optional> config; @@ -135,6 +136,10 @@ class SmartContract : public td::CntObject { this->vm_log_verbosity_level = vm_log_verbosity_level; return std::move(*this); } + Args&& set_debug_enabled(bool debug_enabled) { + this->debug_enabled = debug_enabled; + return std::move(*this); + } td::Result get_method_id() const { if (!method_id) { diff --git a/crypto/test/fift.cpp b/crypto/test/fift.cpp index 466ad4e7..098c5561 100644 --- a/crypto/test/fift.cpp +++ b/crypto/test/fift.cpp @@ -110,3 +110,23 @@ TEST(Fift, test_sort) { TEST(Fift, test_sort2) { run_fift("sort2.fif"); } + +TEST(Fift, test_hmap) { + run_fift("hmap.fif"); +} + +TEST(Fift, test_disasm) { + run_fift("disasm.fif"); +} + +TEST(Fift, test_fiftext) { + run_fift("fift-ext.fif"); +} + +TEST(Fift, test_namespaces) { + run_fift("namespaces.fif"); +} + +TEST(Fift, test_asm_nested_program) { + run_fift("asm-nested-program.fif"); +} diff --git a/crypto/test/fift/asm-nested-program.fif b/crypto/test/fift/asm-nested-program.fif new file mode 100644 index 00000000..ef604d72 --- /dev/null +++ b/crypto/test/fift/asm-nested-program.fif @@ -0,0 +1,93 @@ +"Asm.fif" include + +// Four programs: +// 4 contains 2 and 3, 2 contains 1 + +// Program #1 +PROGRAM{ + DECLPROC foo1 + DECLPROC foo2 + DECLPROC foo3 + DECLPROC main + foo1 PROC:<{ MUL INC }> + foo2 PROCINLINE:<{ PLDREF }> + foo3 PROC:<{ CTOS foo2 INLINECALLDICT CTOS 32 PLDU }> + main PROC:<{ 0 PUSHINT }> +}END>c constant code-1 + +// Program #2 +PROGRAM{ + DECLPROC foo3 + DECLPROC foo4 + DECLPROC main + foo3 PROC:<{ code-1 PUSHREF }> + foo4 PROC:<{ CTOS 8 PLDU }> + main PROC:<{ foo3 CALLDICT foo4 CALLDICT NEWC ROT STUX }> +}END>c constant code-2 + +// Program #3 +PROGRAM{ + DECLPROC foo1 + DECLPROC foo4 + DECLPROC main + foo1 PROC:<{ DUP 137 PUSHINT MUL PAIR }> + foo4 PROC:<{ UNPAIR SWAP DIV }> + main PROC:<{ 70 PUSHINT DIV }> +}END>c constant code-3 + +// Program #4 +PROGRAM{ + DECLPROC foo2 + DECLPROC foo3 + DECLPROC foo5 + DECLPROC main + foo2 PROC:<{ code-2 PUSHREF }> + foo3 PROC:<{ code-3 PUSHREF }> + foo5 PROC:<{ foo2 CALLDICT CTOS 8 PLDU 1 RSHIFT# }> + main PROC:<{ foo5 CALLDICT 5 MULCONST }> +}END>c + +.dump cr + +// Program #4, nested +PROGRAM{ + DECLPROC foo2 + DECLPROC foo3 + DECLPROC foo5 + DECLPROC main + foo2 PROC:<{ + PROGRAM{ + DECLPROC foo3 + DECLPROC foo4 + DECLPROC main + foo3 PROC:<{ + PROGRAM{ + DECLPROC foo1 + DECLPROC foo2 + DECLPROC foo3 + DECLPROC main + foo1 PROC:<{ MUL INC }> + foo2 PROCINLINE:<{ PLDREF }> + foo3 PROC:<{ CTOS foo2 INLINECALLDICT CTOS 32 PLDU }> + main PROC:<{ 0 PUSHINT }> + }END>c PUSHREF + }> + foo4 PROC:<{ CTOS 8 PLDU }> + main PROC:<{ foo3 CALLDICT foo4 CALLDICT NEWC ROT STUX }> + }END>c PUSHREF + }> + foo3 PROC:<{ + PROGRAM{ + DECLPROC foo1 + DECLPROC foo4 + DECLPROC main + foo1 PROC:<{ DUP 137 PUSHINT MUL PAIR }> + foo4 PROC:<{ UNPAIR SWAP DIV }> + main PROC:<{ 70 PUSHINT DIV }> + }END>c PUSHREF + }> + foo5 PROC:<{ foo2 CALLDICT CTOS 8 PLDU 1 RSHIFT# }> + main PROC:<{ foo5 CALLDICT 5 MULCONST }> +}END>c + +.dump cr \ No newline at end of file diff --git a/crypto/test/fift/disasm.fif b/crypto/test/fift/disasm.fif new file mode 100644 index 00000000..8d9f31ac --- /dev/null +++ b/crypto/test/fift/disasm.fif @@ -0,0 +1,70 @@ +"Disasm.fif" include +"Asm.fif" include + +<{ + IF:<{ + 123456789 PUSHINT + }>ELSE<{ + x{12345} PUSHSLICE + }> + WHILE:<{ ADD }>DO<{ + 10 PUSHINT REPEAT:<{ + CONT:<{ NOP }> + CONT:<{ }> + }> + }> +}>s +disasm cr + +x{007A7} disasm cr // Cannot disassemble: x{7} + +<{ + SWAP + s0 s10 XCHG s0 100 s() XCHG + s5 PUSH s6 POP + s4 s10 PUSH2 + 5 10 BLKSWAP + c4 PUSH c5 POP +}>s dup dup +disasm cr +std-disasm disasm cr +stack-disasm show-vm-code disasm cr +hide-vm-code + +<{ + 1 INT 123456789 INT 123456789123456789123456789 INT + PUSHREF + x{567} PUSHSLICE + 10 ADDCONST + DIV DIVR DIVC + RSHIFTC 10 RSHIFTC# + 20 MODPOW2# + 30 MULRSHIFTR# + LSHIFTDIVC 40 LSHIFT#DIVR +}>s +disasm cr + +PROGRAM{ + 11 DECLMETHOD foo1 + 12 DECLMETHOD foo2 + 13 DECLMETHOD foo3 + DECLPROC main + foo1 PROC:<{ + 123 ADDCONST + }> + foo2 PROC:<{ + OVER + 5 EQINT + IFJMP:<{ + NIP + }> + MUL + }> + foo3 PROC:<{ + ADD + foo2 CALLDICT + }> + main PROC:<{ + }> +}END>s +disasm \ No newline at end of file diff --git a/crypto/test/fift/fift-ext.fif b/crypto/test/fift/fift-ext.fif new file mode 100644 index 00000000..b0b08c12 --- /dev/null +++ b/crypto/test/fift/fift-ext.fif @@ -0,0 +1,107 @@ +"FiftExt.fif" include + +// if then +{ + if{ 0> }then{ "Yes" } +} : run +5 run type cr // Yes +-5 run .s // nothing + +// if then else +{ + if{ 0> }then{ "Yes" }else{ "No" } +} : run +5 run type ." " // Yes +-5 run type cr // No + +// if then elseif else +{ + dup dup if{ 20 > }then{ 2drop "AAA" }elseif{ 10 > }then{ drop "BBB" }elseif{ 0> }then{ "CCC" }else{ "DDD" } +} : run +25 run type ." " // AAA +15 run type ." " // BBB +5 run type ." " // CCC +-5 run type cr // DDD + +// while do +1 +while{ dup 100 < }do{ + dup . + 2 * +} +cr // 1 2 4 8 16 32 64 +drop + +// repeat until +1 +repeat{ + dup . + 3 * +}until{ dup 500 > } +cr // 1 3 9 27 81 243 +drop + +// def +def foo1 { * + } +5 10 20 foo1 . // 205 +6 100 100 foo1 . cr // 10006 + +// defrec +defrec fib { + if{ dup 2 < }then{ + drop 1 + }else{ + dup 1- fib swap 2- fib + + } +} +8 fib . // 34 +9 fib . // 55 +20 fib . cr // 10946 + +// [[ ... ]] +def foo2 { [[ ."Exec" cr 5 5 * ]] + } +."Calling foo2: " +100 foo2 . 200 foo2 . cr + +// Nested blocks +def sqr { dup * } +def power { + 1 -rot + while{ dup 0> }do{ + if{ dup 1 and }then{ + -rot tuck * swap rot + } + 2/ + if{ dup 0> }then{ + swap sqr swap + } + } + 2drop +} + +3 0 power . // 1 +3 1 power . // 3 +3 4 power . // 81 +3 12 power . // 531441 +3 50 power . // 717897987691852588770249 +cr + +0 while{ dup 2 < }do{ + ."A" + 0 while{ dup 2 < }do{ + ."B" + 0 while{ dup 2 < }do{ + ."C" + 0 while{ dup 2 < }do{ + ."D" + 0 while{ dup 2 < }do{ + ."E" + 0 while{ dup 2 < }do{ + ."F" + 1+ } drop + 1+ } drop + 1+ } drop + 1+ } drop + 1+ } drop +1+ } drop +cr // ABCDEFFEFFDEFFEFFCDEFFEFFDEFFEFFBCDEFFEFFDEFFEFFCDEFFEFFDEFFEFFABCDEFFEFFDEFFEFFCDEFFEFFDEFFEFFBCDEFFEFFDEFFEFFCDEFFEFFDEFFEFF \ No newline at end of file diff --git a/crypto/test/fift/hmap.fif b/crypto/test/fift/hmap.fif new file mode 100644 index 00000000..190be938 --- /dev/null +++ b/crypto/test/fift/hmap.fif @@ -0,0 +1,69 @@ +hmapnew .s // (null) + +// Get from empty hmap +dup 10 swap hmap@ null? not abort"expected null" +dup "abc" swap hmap@ null? not abort"expected null" +dup 10 swap hmap@? abort"expected 0" +dup "abc" swap hmap@? abort"expected 0" +dup hmapempty? not abort"expected -1" +.s cr // (null) + +// Insert values +"value-1" 123 rot hmap!+ +"value-2" "Abc" rot hmap!+ +"value-3" "xyz" rot hmap!+ +"value-4" B{78797A} rot hmap!+ +"value-5" `xyz rot hmap!+ +"abcdefg" null rot hmap!+ // No effect! +dup hmapempty? abort"expected 0" + +// Get values +dup 123 swap hmap@ type cr // value-1 +dup "Abc" swap hmap@ type cr // value-2 +dup "xyz" swap hmap@ type cr // value-3 +dup B{78797A} swap hmap@ type cr // value-4 +dup `xyz swap hmap@ type cr // value-5 +dup 123 swap hmap@? . type cr // -1 value-1 +dup "Abc" swap hmap@? . type cr // -1 value-2 +dup "xyz" swap hmap@? . type cr // -1 value-3 +dup B{78797A} swap hmap@? . type cr // -1 value-4 +dup `xyz swap hmap@? . type cr // -1 value-5 +// Get null +dup 1234 swap hmap@ null? not abort"expected null" +dup null swap hmap@ null? not abort"expected null" +dup 1234 swap hmap@? abort"expected 0" +dup null swap hmap@? abort"expected 0" +cr + +// hmapforeach +constant hmap +hmap { .s drop drop -1 } hmapforeach +.s drop +3 hmap { .s drop drop 1- dup } hmapforeach +.s cr drop drop hmap + +// hmap! (insert) and hmap!+ (replace) +"value-11" 123 rot hmap!+ +"value-66" 567 rot hmap!+ +"value-33" "xyz" rot hmap! +"value-77" "qwe" rot hmap! // No effect! +dup 123 swap hmap@ type cr // value-11 +dup 567 swap hmap@ type cr // value-66 +dup "xyz" swap hmap@ type cr // value-33 +dup "qwe" swap hmap@ null? not abort"null expected" +cr + +// Delete +567 swap hmap- +789 swap hmap- +"qwe" swap hmap-? . // -1 +"rty" swap hmap-? . // 0 +`xyz swap hmap@- type // value-5 +`zyx swap hmap@- null? not abort"null expected" +null 123 rot hmap! // same as hmap- +null 321 rot hmap! +null `xyz rot hmap!+ +cr +constant hmap +hmap { .s drop drop -1 } hmapforeach +drop diff --git a/crypto/test/fift/namespaces.fif b/crypto/test/fift/namespaces.fif new file mode 100644 index 00000000..3f10f608 --- /dev/null +++ b/crypto/test/fift/namespaces.fif @@ -0,0 +1,29 @@ +namespace My + +"a" constant a +"b" constant b +"c" constant c + +a b c .s { drop } 3 times // "a" "b" "c" + +My definitions +"b-my" constant b +"c-my" constant c +"d-my" constant d + +a b c d .s { drop } 4 times // "a" "b-my" "c-my" "d-my" + +Fift definitions +a b c .s { drop } 3 times // "a" "b-my" "c-my" "d-my" + +My b My c My d .s { drop } 3 times // "b-my" "c-my" "d-my" +a b c .s { drop } 3 times // "a" "b" "c" "d" + +My definitions +a b c d .s { drop } 4 times // "a" "b-my" "c-my" "d-my" +Fift a Fift b Fift c d .s { drop } 4 times // "a" "b" "c" "d-my" + +Fift definitions +cr +My-wordlist @ +{ drop type -1 } hmapforeach drop cr // "b " "d " "c " \ No newline at end of file diff --git a/crypto/vm/arithops.cpp b/crypto/vm/arithops.cpp index 2831944b..24bb8a48 100644 --- a/crypto/vm/arithops.cpp +++ b/crypto/vm/arithops.cpp @@ -38,8 +38,8 @@ int exec_push_tinyint4(VmState* st, unsigned args) { std::string dump_push_tinyint4(CellSlice&, unsigned args) { int x = (int)((args + 5) & 15) - 5; - std::ostringstream os{"PUSHINT "}; - os << x; + std::ostringstream os; + os << "PUSHINT " << x; return os.str(); } @@ -53,8 +53,8 @@ int exec_push_tinyint8(VmState* st, unsigned args) { std::string dump_op_tinyint8(const char* op_prefix, CellSlice&, unsigned args) { int x = (signed char)args; - std::ostringstream os{op_prefix}; - os << x; + std::ostringstream os; + os << op_prefix << x; return os.str(); } @@ -68,8 +68,8 @@ int exec_push_smallint(VmState* st, unsigned args) { std::string dump_push_smallint(CellSlice&, unsigned args) { int x = (short)args; - std::ostringstream os{"PUSHINT "}; - os << x; + std::ostringstream os; + os << "PUSHINT " << x; return os.str(); } @@ -93,8 +93,8 @@ std::string dump_push_int(CellSlice& cs, unsigned args, int pfx_bits) { } cs.advance(pfx_bits); td::RefInt256 x = cs.fetch_int256(3 + l * 8); - std::ostringstream os{"PUSHINT "}; - os << x; + std::ostringstream os; + os << "PUSHINT " << x; return os.str(); } @@ -302,7 +302,7 @@ std::string dump_divmod(CellSlice&, unsigned args, bool quiet) { if (quiet) { s = "Q" + s; } - return s + "FRC"[round_mode]; + return round_mode ? s + "FRC"[round_mode] : s; } int exec_shrmod(VmState* st, unsigned args, int mode) { @@ -352,28 +352,28 @@ std::string dump_shrmod(CellSlice&, unsigned args, int mode) { if (!(args & 12) || round_mode == 3) { return ""; } - std::string s; + std::ostringstream os; + if (mode & 1) { + os << 'Q'; + } switch (args & 12) { case 4: - s = "RSHIFT"; + os << "RSHIFT"; break; case 8: - s = "MODPOW2"; + os << "MODPOW2"; break; case 12: - s = "RSHIFTMOD"; + os << "RSHIFTMOD"; break; } - if (mode & 1) { - s = "Q" + s; + if (round_mode) { + os << "FRC"[round_mode]; } - s += "FRC"[round_mode]; if (mode & 2) { - char buff[8]; - sprintf(buff, " %d", y); - s += buff; + os << ' ' << y; } - return s; + return os.str(); } int exec_muldivmod(VmState* st, unsigned args, int quiet) { @@ -417,7 +417,7 @@ std::string dump_muldivmod(CellSlice&, unsigned args, bool quiet) { if (quiet) { s = "Q" + s; } - return s + "FRC"[round_mode]; + return round_mode ? s + "FRC"[round_mode] : s; } int exec_mulshrmod(VmState* st, unsigned args, int mode) { @@ -474,28 +474,28 @@ std::string dump_mulshrmod(CellSlice&, unsigned args, int mode) { if (!(args & 12) || round_mode == 3) { return ""; } - std::string s; + std::ostringstream os; + if (mode & 1) { + os << 'Q'; + } switch (args & 12) { case 4: - s = "MULRSHIFT"; + os << "MULRSHIFT"; break; case 8: - s = "MULMODPOW2"; + os << "MULMODPOW2"; break; case 12: - s = "MULRSHIFTMOD"; + os << "MULRSHIFTMOD"; break; } - if (mode & 1) { - s = "Q" + s; + if (round_mode) { + os << "FRC"[round_mode]; } - s += "FRC"[round_mode]; if (mode & 2) { - char buff[8]; - sprintf(buff, " %d", y); - s += buff; + os << ' ' << y; } - return s; + return os.str(); } int exec_shldivmod(VmState* st, unsigned args, int mode) { @@ -542,19 +542,25 @@ int exec_shldivmod(VmState* st, unsigned args, int mode) { return 0; } -std::string dump_shldivmod(CellSlice&, unsigned args, bool quiet) { +std::string dump_shldivmod(CellSlice&, unsigned args, int mode) { + int y = -1; + if (mode & 2) { + y = (args & 0xff) + 1; + args >>= 8; + } int round_mode = (int)(args & 3); if (!(args & 12) || round_mode == 3) { return ""; } - std::string s = (args & 4) ? "LSHIFTDIV" : "LSHIFT"; - if (args & 8) { - s += "MOD"; + std::ostringstream os; + os << (mode & 1 ? "Q" : "") << (args & 4 ? "LSHIFTDIV" : "LSHIFT") << (args & 8 ? "MOD" : ""); + if (round_mode) { + os << "FRC"[round_mode]; } - if (quiet) { - s = "Q" + s; + if (y >= 0) { + os << ' ' << y; } - return s + "FRC"[round_mode]; + return os.str(); } void register_div_ops(OpcodeTable& cp0) { diff --git a/crypto/vm/box.hpp b/crypto/vm/box.hpp index 68b131ce..0d676cae 100644 --- a/crypto/vm/box.hpp +++ b/crypto/vm/box.hpp @@ -30,7 +30,7 @@ class Box : public td::CntObject { Box(const Box&) = default; Box(Box&&) = default; template - Box(Args... args) : data_{std::move(args...)} { + Box(Args&&... args) : data_{std::forward(args)...} { } ~Box() override = default; Box(const StackEntry& data) : data_(data) { diff --git a/crypto/vm/cellops.cpp b/crypto/vm/cellops.cpp index 1dfb69fb..9e10e072 100644 --- a/crypto/vm/cellops.cpp +++ b/crypto/vm/cellops.cpp @@ -103,7 +103,8 @@ std::string dump_push_slice_common(CellSlice& cs, unsigned data_bits, unsigned r cs.advance(pfx_bits); auto slice = cs.fetch_subslice(data_bits, refs); slice.unique_write().remove_trailing(); - std::ostringstream os{name}; + std::ostringstream os; + os << name; slice->dump_hex(os, 1, false); return os.str(); } @@ -188,7 +189,8 @@ std::string dump_push_cont(CellSlice& cs, unsigned args, int pfx_bits) { } cs.advance(pfx_bits); auto slice = cs.fetch_subslice(data_bits, refs); - std::ostringstream os{"PUSHCONT "}; + std::ostringstream os; + os << "PUSHCONT "; slice->dump_hex(os, 1, false); return os.str(); } @@ -219,7 +221,8 @@ std::string dump_push_cont_simple(CellSlice& cs, unsigned args, int pfx_bits) { } cs.advance(pfx_bits); auto slice = cs.fetch_subslice(data_bits); - std::ostringstream os{"PUSHCONT "}; + std::ostringstream os; + os << "PUSHCONT "; slice->dump_hex(os, 1, false); return os.str(); } @@ -1060,8 +1063,8 @@ int exec_load_int_fixed2(VmState* st, unsigned args) { } std::string dump_load_int_fixed2(CellSlice&, unsigned args) { - std::ostringstream os{args & 0x200 ? "PLD" : "LD"}; - os << (args & 0x100 ? 'U' : 'I'); + std::ostringstream os; + os << (args & 0x200 ? "PLD" : "LD") << (args & 0x100 ? 'U' : 'I'); if (args & 0x400) { os << 'Q'; } @@ -1081,9 +1084,9 @@ int exec_preload_uint_fixed_0e(VmState* st, unsigned args) { } std::string dump_preload_uint_fixed_0e(CellSlice&, unsigned args) { - std::ostringstream os{"PLDUZ "}; + std::ostringstream os; unsigned bits = ((args & 7) + 1) << 5; - os << bits; + os << "PLDUZ " << bits; return os.str(); } @@ -1108,7 +1111,8 @@ int exec_load_slice_fixed2(VmState* st, unsigned args) { std::string dump_load_slice_fixed2(CellSlice&, unsigned args) { unsigned bits = (args & 0xff) + 1; - std::ostringstream os{args & 0x100 ? "PLDSLICE" : "LDSLICE"}; + std::ostringstream os; + os << (args & 0x100 ? "PLDSLICE" : "LDSLICE"); if (args & 0x200) { os << 'Q'; } diff --git a/crypto/vm/contops.cpp b/crypto/vm/contops.cpp index c3cb36bb..a6a44ad3 100644 --- a/crypto/vm/contops.cpp +++ b/crypto/vm/contops.cpp @@ -378,8 +378,8 @@ int exec_if_bit_jmp(VmState* st, unsigned args) { } std::string dump_if_bit_jmp(CellSlice& cs, unsigned args) { - std::ostringstream os{args & 0x20 ? "IFN" : " IF"}; - os << "BITJMP " << (args & 0x1f); + std::ostringstream os; + os << "IF" << (args & 0x20 ? "N" : "") << "BITJMP " << (args & 0x1f); return os.str(); } @@ -408,8 +408,8 @@ std::string dump_if_bit_jmpref(CellSlice& cs, unsigned args, int pfx_bits) { } cs.advance(pfx_bits); cs.advance_refs(1); - std::ostringstream os{args & 0x20 ? "IFN" : " IF"}; - os << "BITJMPREF " << (args & 0x1f); + std::ostringstream os; + os << "IF" << (args & 0x20 ? "N" : "") << "BITJMPREF " << (args & 0x1f); return os.str(); } @@ -597,8 +597,8 @@ int exec_setcontargs(VmState* st, unsigned args) { std::string dump_setcontargs(CellSlice& cs, unsigned args, const char* name) { int copy = (args >> 4) & 15, more = ((args + 1) & 15) - 1; - std::ostringstream os{name}; - os << ' ' << copy << ',' << more; + std::ostringstream os; + os << name << ' ' << copy << ',' << more; return os.str(); } @@ -1065,8 +1065,8 @@ std::string dump_throw_any(CellSlice& cs, unsigned args) { bool has_param = args & 1; bool has_cond = args & 6; bool throw_cond = args & 2; - std::ostringstream os{has_param ? "THROWARG" : "THROW"}; - os << "ANY"; + std::ostringstream os; + os << "THROW" << (has_param ? "ARG" : "") << "ANY"; if (has_cond) { os << (throw_cond ? "IF" : "IFNOT"); } diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 5441feea..0d1d099f 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -138,8 +138,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (cell->get_virtualization() != 0) { return; } - //LOG(ERROR) << "INC"; - //CellSlice(cell, nullptr).print_rec(std::cout); to_inc_.push_back(cell); } void dec(const Ref &cell) override { @@ -149,8 +147,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (cell->get_virtualization() != 0) { return; } - //LOG(ERROR) << "DEC"; - //CellSlice(cell, nullptr).print_rec(std::cout); to_dec_.push_back(cell); } @@ -167,25 +163,20 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (is_prepared_for_commit()) { return td::Status::OK(); } - //LOG(ERROR) << "dfs_new_cells_in_db"; for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells_in_db(new_cell_info); } - //return td::Status::OK(); - //LOG(ERROR) << "dfs_new_cells"; for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells(new_cell_info); } - //LOG(ERROR) << "dfs_old_cells"; for (auto &old_cell : to_dec_) { auto &old_cell_info = get_cell_info(old_cell); dfs_old_cells(old_cell_info); } - //LOG(ERROR) << "save_diff_prepare"; save_diff_prepare(); to_inc_.clear(); @@ -363,7 +354,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat info.was = true; visited_.push_back(&info); } - //LOG(ERROR) << "dfs new " << td::format::escaped(info.cell->hash()); if (info.was_dfs_new_cells) { return; @@ -384,7 +374,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat info.was = true; visited_.push_back(&info); } - //LOG(ERROR) << "dfs old " << td::format::escaped(info.cell->hash()); load_cell(info); @@ -405,7 +394,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } void save_diff(CellStorer &storer) { - //LOG(ERROR) << hash_table_.size(); for (auto info_ptr : visited_) { save_cell(*info_ptr, storer); } @@ -414,7 +402,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat void save_cell_prepare(CellInfo &info) { if (info.refcnt_diff == 0) { - //CellSlice(info.cell, nullptr).print_rec(std::cout); return; } load_cell(info); @@ -450,15 +437,11 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (info.db_refcnt == 0) { CHECK(info.in_db); - //LOG(ERROR) << "ERASE"; - //CellSlice(NoVm(), info.cell).print_rec(std::cout); storer.erase(info.cell->get_hash().as_slice()); info.in_db = false; hash_table_.erase(info.cell->get_hash().as_slice()); guard.dismiss(); } else { - //LOG(ERROR) << "SAVE " << info.db_refcnt; - //CellSlice(NoVm(), info.cell).print_rec(std::cout); auto loaded_cell = info.cell->load_cell().move_as_ok(); storer.set(info.db_refcnt, *loaded_cell.data_cell); info.in_db = true; @@ -482,7 +465,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat CHECK(cell->is_loaded()); vm::CellSlice cs(vm::NoVm{}, cell); // FIXME for (unsigned i = 0; i < cs.size_refs(); i++) { - //LOG(ERROR) << "---> " << td::format::escaped(cell->ref(i)->hash()); f(get_cell_info(cs.prefetch_ref(i))); } } diff --git a/crypto/vm/dictops.cpp b/crypto/vm/dictops.cpp index c798b333..d0ea8daa 100644 --- a/crypto/vm/dictops.cpp +++ b/crypto/vm/dictops.cpp @@ -172,7 +172,8 @@ int exec_load_dict(VmState* st, unsigned args) { } std::string dump_dictop(unsigned args, const char* name) { - std::ostringstream os{"DICT"}; + std::ostringstream os; + os << "DICT"; if (args & 4) { os << (args & 2 ? 'U' : 'I'); } @@ -184,7 +185,8 @@ std::string dump_dictop(unsigned args, const char* name) { } std::string dump_dictop2(unsigned args, const char* name) { - std::ostringstream os{"DICT"}; + std::ostringstream os; + os << "DICT"; if (args & 2) { os << (args & 1 ? 'U' : 'I'); } @@ -193,7 +195,8 @@ std::string dump_dictop2(unsigned args, const char* name) { } std::string dump_subdictop2(unsigned args, const char* name) { - std::ostringstream os{"SUBDICT"}; + std::ostringstream os; + os << "SUBDICT"; if (args & 2) { os << (args & 1 ? 'U' : 'I'); } @@ -508,7 +511,8 @@ int exec_dict_getmin(VmState* st, unsigned args) { } std::string dump_dictop_getnear(CellSlice& cs, unsigned args) { - std::ostringstream os{"DICT"}; + std::ostringstream os; + os << "DICT"; if (args & 8) { os << (args & 4 ? 'U' : 'I'); } @@ -637,8 +641,8 @@ std::string dump_push_const_dict(CellSlice& cs, int pfx_bits, const char* name) cs.advance(pfx_bits - 11); auto slice = cs.fetch_subslice(1, 1); int n = (int)cs.fetch_ulong(10); - std::ostringstream os{name}; - os << ' ' << n << " ("; + std::ostringstream os; + os << name << ' ' << n << " ("; slice->dump_hex(os, false); os << ')'; return os.str(); diff --git a/crypto/vm/opctable.cpp b/crypto/vm/opctable.cpp index d4f0f3e9..0ee33385 100644 --- a/crypto/vm/opctable.cpp +++ b/crypto/vm/opctable.cpp @@ -357,32 +357,32 @@ using namespace std::placeholders; dump_arg_instr_func_t dump_1sr(std::string prefix, std::string suffix) { return [prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << (args & 15) << suffix; + std::ostringstream os; + os << prefix << 's' << (args & 15) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_1sr_l(std::string prefix, std::string suffix) { return [prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << (args & 255) << suffix; + std::ostringstream os; + os << prefix << 's' << (args & 255) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_2sr(std::string prefix, std::string suffix) { return [prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << ((args >> 4) & 15) << ",s" << (args & 15) << suffix; + std::ostringstream os; + os << prefix << 's' << ((args >> 4) & 15) << ",s" << (args & 15) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_2sr_adj(unsigned adj, std::string prefix, std::string suffix) { return [adj, prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << (int)((args >> 4) & 15) - (int)((adj >> 4) & 15) << ",s" << (int)(args & 15) - (int)(adj & 15) + std::ostringstream os; + os << prefix << 's' << (int)((args >> 4) & 15) - (int)((adj >> 4) & 15) << ",s" << (int)(args & 15) - (int)(adj & 15) << suffix; return os.str(); }; @@ -390,16 +390,16 @@ dump_arg_instr_func_t dump_2sr_adj(unsigned adj, std::string prefix, std::string dump_arg_instr_func_t dump_3sr(std::string prefix, std::string suffix) { return [prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << ((args >> 8) & 15) << ",s" << ((args >> 4) & 15) << ",s" << (args & 15) << suffix; + std::ostringstream os; + os << prefix << 's' << ((args >> 8) & 15) << ",s" << ((args >> 4) & 15) << ",s" << (args & 15) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_3sr_adj(unsigned adj, std::string prefix, std::string suffix) { return [adj, prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << (int)((args >> 8) & 15) - (int)((adj >> 8) & 15) << ",s" + std::ostringstream os; + os << prefix << 's' << (int)((args >> 8) & 15) - (int)((adj >> 8) & 15) << ",s" << (int)((args >> 4) & 15) - (int)((adj >> 4) & 15) << ",s" << (int)(args & 15) - (int)(adj & 15) << suffix; return os.str(); }; @@ -407,40 +407,40 @@ dump_arg_instr_func_t dump_3sr_adj(unsigned adj, std::string prefix, std::string dump_arg_instr_func_t dump_1c(std::string prefix, std::string suffix) { return [prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << (args & 15) << suffix; + std::ostringstream os; + os << prefix << (args & 15) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_1c_l_add(int adj, std::string prefix, std::string suffix) { return [adj, prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << (int)(args & 255) + adj << suffix; + std::ostringstream os; + os << prefix << (int)(args & 255) + adj << suffix; return os.str(); }; } dump_arg_instr_func_t dump_1c_and(unsigned mask, std::string prefix, std::string suffix) { return [mask, prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << (args & mask) << suffix; + std::ostringstream os; + os << prefix << (args & mask) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_2c(std::string prefix, std::string interfix, std::string suffix) { return [prefix, interfix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << ((args >> 4) & 15) << interfix << (args & 15) << suffix; + std::ostringstream os; + os << prefix << ((args >> 4) & 15) << interfix << (args & 15) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_2c_add(unsigned add, std::string prefix, std::string interfix, std::string suffix) { return [add, prefix, interfix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << ((args >> 4) & 15) + ((add >> 4) & 15) << interfix << (args & 15) + (add & 15) << suffix; + std::ostringstream os; + os << prefix << ((args >> 4) & 15) + ((add >> 4) & 15) << interfix << (args & 15) + (add & 15) << suffix; return os.str(); }; } diff --git a/crypto/vm/stack.cpp b/crypto/vm/stack.cpp index 9a17c8e2..e82b25de 100644 --- a/crypto/vm/stack.cpp +++ b/crypto/vm/stack.cpp @@ -908,23 +908,29 @@ bool Stack::serialize(vm::CellBuilder& cb, int mode) const { if (vsi && !vsi->register_op()) { return false; } - // vm_stack#_ depth:(## 24) stack:(VmStackList depth) = VmStack; - unsigned n = depth(); - if (!cb.store_ulong_rchk_bool(n, 24)) { // vm_stack#_ depth:(## 24) - return false; - } - if (!n) { - return true; - } - vm::CellBuilder cb2; - Ref rest = cb2.finalize(); // vm_stk_nil#_ = VmStackList 0; - for (unsigned i = 0; i < n - 1; i++) { - // vm_stk_cons#_ {n:#} rest:^(VmStackList n) tos:VmStackValue = VmStackList (n + 1); - if (!(cb2.store_ref_bool(std::move(rest)) && stack[i].serialize(cb2, mode) && cb2.finalize_to(rest))) { + try { + // vm_stack#_ depth:(## 24) stack:(VmStackList depth) = VmStack; + unsigned n = depth(); + if (!cb.store_ulong_rchk_bool(n, 24)) { // vm_stack#_ depth:(## 24) return false; } + if (!n) { + return true; + } + vm::CellBuilder cb2; + Ref rest = cb2.finalize(); // vm_stk_nil#_ = VmStackList 0; + for (unsigned i = 0; i < n - 1; i++) { + // vm_stk_cons#_ {n:#} rest:^(VmStackList n) tos:VmStackValue = VmStackList (n + 1); + if (!(cb2.store_ref_bool(std::move(rest)) && stack[i].serialize(cb2, mode) && cb2.finalize_to(rest))) { + return false; + } + } + return cb.store_ref_bool(std::move(rest)) && stack[n - 1].serialize(cb, mode); + } catch (CellBuilder::CellCreateError) { + return false; + } catch (CellBuilder::CellWriteError) { + return false; } - return cb.store_ref_bool(std::move(rest)) && stack[n - 1].serialize(cb, mode); } bool Stack::deserialize(vm::CellSlice& cs, int mode) { diff --git a/crypto/vm/stack.hpp b/crypto/vm/stack.hpp index 2de7c1e4..bfc9e7ac 100644 --- a/crypto/vm/stack.hpp +++ b/crypto/vm/stack.hpp @@ -103,6 +103,9 @@ class StackEntry { } StackEntry(td::RefInt256 int_ref) : ref(std::move(int_ref)), tp(t_int) { } + StackEntry(Ref> str_ref, bool bytes = false) + : ref(std::move(str_ref)), tp(bytes ? t_bytes : t_string) { + } StackEntry(std::string str, bool bytes = false) : ref(), tp(bytes ? t_bytes : t_string) { ref = Ref>{true, std::move(str)}; } diff --git a/crypto/vm/stackops.cpp b/crypto/vm/stackops.cpp index 0f5be6d4..a52a90e2 100644 --- a/crypto/vm/stackops.cpp +++ b/crypto/vm/stackops.cpp @@ -75,8 +75,8 @@ std::string dump_xchg(CellSlice&, unsigned args) { if (!x || x >= y) { return ""; } - std::ostringstream os{"XCHG s"}; - os << x << ",s" << y; + std::ostringstream os; + os << "XCHG s" << x << ",s" << y; return os.str(); } @@ -92,7 +92,7 @@ int exec_xchg1(VmState* st, unsigned args) { int exec_dup(VmState* st) { Stack& stack = st->get_stack(); - VM_LOG(st) << "execute DUP\n"; + VM_LOG(st) << "execute DUP"; stack.check_underflow(1); stack.push(stack.fetch(0)); return 0; @@ -100,7 +100,7 @@ int exec_dup(VmState* st) { int exec_over(VmState* st) { Stack& stack = st->get_stack(); - VM_LOG(st) << "execute OVER\n"; + VM_LOG(st) << "execute OVER"; stack.check_underflow(2); stack.push(stack.fetch(1)); return 0; @@ -126,7 +126,7 @@ int exec_push_l(VmState* st, unsigned args) { int exec_drop(VmState* st) { Stack& stack = st->get_stack(); - VM_LOG(st) << "execute DROP\n"; + VM_LOG(st) << "execute DROP"; stack.check_underflow(1); stack.pop(); return 0; @@ -134,7 +134,7 @@ int exec_drop(VmState* st) { int exec_nip(VmState* st) { Stack& stack = st->get_stack(); - VM_LOG(st) << "execute NIP\n"; + VM_LOG(st) << "execute NIP"; stack.check_underflow(2); stack.pop(stack[1]); return 0; diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 212a6fae..f729105f 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -170,7 +170,7 @@ ton::tl_object_ptr Config::tl() const { return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, diff --git a/dht/dht-remote-node.cpp b/dht/dht-remote-node.cpp index 653de256..1273750c 100644 --- a/dht/dht-remote-node.cpp +++ b/dht/dht-remote-node.cpp @@ -140,13 +140,7 @@ adnl::AdnlNodeIdFull DhtRemoteNode::get_full_id() const { td::Result> DhtRemoteNode::create(DhtNode node, td::uint32 max_missed_pings, td::int32 our_network_id) { - TRY_RESULT(enc, node.adnl_id().pubkey().create_encryptor()); - auto tl = node.tl(); - auto sig = std::move(tl->signature_); - - TRY_STATUS_PREFIX(enc->check_signature(serialize_tl_object(tl, true).as_slice(), sig.as_slice()), - "bad node signature: "); - + TRY_STATUS(node.check_signature()); return std::make_unique(std::move(node), max_missed_pings, our_network_id); } diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index 2fb27519..4e67d32c 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -51,5 +51,8 @@ if (USE_EMSCRIPTEN) target_link_options(emulator-emscripten PRIVATE -sENVIRONMENT=web) target_link_options(emulator-emscripten PRIVATE -sFILESYSTEM=0) target_link_options(emulator-emscripten PRIVATE -fexceptions) + if (USE_EMSCRIPTEN_NO_WASM) + target_link_options(emulator-emscripten PRIVATE -sWASM=0) + endif() target_compile_options(emulator-emscripten PRIVATE -fexceptions) endif() diff --git a/emulator/emulator-emscripten.cpp b/emulator/emulator-emscripten.cpp index f0f2c903..881664bd 100644 --- a/emulator/emulator-emscripten.cpp +++ b/emulator/emulator-emscripten.cpp @@ -12,6 +12,7 @@ struct TransactionEmulationParams { uint64_t lt; td::optional rand_seed_hex; bool ignore_chksig; + bool debug_enabled; }; td::Result decode_transaction_emulation_params(const char* json) { @@ -37,6 +38,9 @@ td::Result decode_transaction_emulation_params(const TRY_RESULT(ignore_chksig, td::get_json_object_bool_field(obj, "ignore_chksig", false)); params.ignore_chksig = ignore_chksig; + TRY_RESULT(debug_enabled, td::get_json_object_bool_field(obj, "debug_enabled", false)); + params.debug_enabled = debug_enabled; + return params; } @@ -51,6 +55,7 @@ struct GetMethodParams { std::string rand_seed_hex; int64_t gas_limit; int method_id; + bool debug_enabled; }; td::Result decode_get_method_params(const char* json) { @@ -95,6 +100,9 @@ td::Result decode_get_method_params(const char* json) { TRY_RESULT(method_id, td::get_json_object_int_field(obj, "method_id", false)); params.method_id = method_id; + TRY_RESULT(debug_enabled, td::get_json_object_bool_field(obj, "debug_enabled", false)); + params.debug_enabled = debug_enabled; + return params; } @@ -123,6 +131,7 @@ const char *emulate(const char *config, const char* libs, int verbosity, const c !transaction_emulator_set_lt(em, decoded_params.lt) || !transaction_emulator_set_unixtime(em, decoded_params.utime) || !transaction_emulator_set_ignore_chksig(em, decoded_params.ignore_chksig) || + !transaction_emulator_set_debug_enabled(em, decoded_params.debug_enabled) || !rand_seed_set) { transaction_emulator_destroy(em); return strdup(R"({"fail":true,"message":"Can't set params"})"); @@ -163,7 +172,8 @@ const char *run_get_method(const char *params, const char* stack, const char* co if ((decoded_params.libs && !tvm_emulator_set_libraries(tvm, decoded_params.libs.value().c_str())) || !tvm_emulator_set_c7(tvm, decoded_params.address.c_str(), decoded_params.unixtime, decoded_params.balance, decoded_params.rand_seed_hex.c_str(), config) || - (decoded_params.gas_limit > 0 && !tvm_emulator_set_gas_limit(tvm, decoded_params.gas_limit))) { + (decoded_params.gas_limit > 0 && !tvm_emulator_set_gas_limit(tvm, decoded_params.gas_limit)) || + !tvm_emulator_set_debug_enabled(tvm, decoded_params.debug_enabled)) { tvm_emulator_destroy(tvm); return strdup(R"({"fail":true,"message":"Can't set params"})"); } diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp index 811316db..31a7b649 100644 --- a/emulator/emulator-extern.cpp +++ b/emulator/emulator-extern.cpp @@ -19,7 +19,8 @@ td::Result cell_to_boc_b64(td::Ref cell) { return td::base64_encode(boc.as_slice()); } -const char *success_response(std::string&& transaction, std::string&& new_shard_account, std::string&& vm_log, td::optional&& actions) { +const char *success_response(std::string&& transaction, std::string&& new_shard_account, std::string&& vm_log, + td::optional&& actions, double elapsed_time) { td::JsonBuilder jb; auto json_obj = jb.enter_object(); json_obj("success", td::JsonTrue()); @@ -31,6 +32,7 @@ const char *success_response(std::string&& transaction, std::string&& new_shard_ } else { json_obj("actions", td::JsonNull()); } + json_obj("elapsed_time", elapsed_time); json_obj.leave(); return strdup(jb.string_builder().as_cslice().c_str()); } @@ -44,13 +46,14 @@ const char *error_response(std::string&& error) { return strdup(jb.string_builder().as_cslice().c_str()); } -const char *external_not_accepted_response(std::string&& vm_log, int vm_exit_code) { +const char *external_not_accepted_response(std::string&& vm_log, int vm_exit_code, double elapsed_time) { td::JsonBuilder jb; auto json_obj = jb.enter_object(); json_obj("success", td::JsonFalse()); json_obj("error", "External message not accepted by smart contract"); json_obj("vm_log", std::move(vm_log)); json_obj("vm_exit_code", vm_exit_code); + json_obj("elapsed_time", elapsed_time); json_obj.leave(); return strdup(jb.string_builder().as_cslice().c_str()); } @@ -142,7 +145,8 @@ const char *transaction_emulator_emulate_transaction(void *transaction_emulator, auto external_not_accepted = dynamic_cast(emulation_result.get()); if (external_not_accepted) { - return external_not_accepted_response(std::move(external_not_accepted->vm_log), external_not_accepted->vm_exit_code); + return external_not_accepted_response(std::move(external_not_accepted->vm_log), external_not_accepted->vm_exit_code, + external_not_accepted->elapsed_time); } auto emulation_success = dynamic_cast(*emulation_result); @@ -168,7 +172,8 @@ const char *transaction_emulator_emulate_transaction(void *transaction_emulator, actions_boc_b64 = actions_boc_b64_result.move_as_ok(); } - return success_response(trans_boc_b64.move_as_ok(), new_shard_account_boc_b64.move_as_ok(), std::move(emulation_success.vm_log), std::move(actions_boc_b64)); + return success_response(trans_boc_b64.move_as_ok(), new_shard_account_boc_b64.move_as_ok(), std::move(emulation_success.vm_log), + std::move(actions_boc_b64), emulation_success.elapsed_time); } bool transaction_emulator_set_unixtime(void *transaction_emulator, uint32_t unixtime) { @@ -244,6 +249,14 @@ bool transaction_emulator_set_libs(void *transaction_emulator, const char* shard return true; } +bool transaction_emulator_set_debug_enabled(void *transaction_emulator, bool debug_enabled) { + auto emulator = static_cast(transaction_emulator); + + emulator->set_debug_enabled(debug_enabled); + + return true; +} + void transaction_emulator_destroy(void *transaction_emulator) { delete static_cast(transaction_emulator); } @@ -332,6 +345,12 @@ bool tvm_emulator_set_gas_limit(void *tvm_emulator, int64_t gas_limit) { return true; } +bool tvm_emulator_set_debug_enabled(void *tvm_emulator, bool debug_enabled) { + auto emulator = static_cast(tvm_emulator); + emulator->set_debug_enabled(debug_enabled); + return true; +} + const char *tvm_emulator_run_get_method(void *tvm_emulator, int method_id, const char *stack_boc) { auto stack_cell = boc_b64_to_cell(stack_boc); if (stack_cell.is_error()) { @@ -393,7 +412,11 @@ const char *tvm_emulator_send_external_message(void *tvm_emulator, const char *m } else { json_obj("missing_library", td::Bits256(result.missing_library).to_hex()); } - json_obj("actions", cell_to_boc_b64(result.actions).move_as_ok()); + if (result.actions.is_null()) { + json_obj("actions", td::JsonNull()); + } else { + json_obj("actions", cell_to_boc_b64(result.actions).move_as_ok()); + } json_obj("new_code", cell_to_boc_b64(result.new_state.code).move_as_ok()); json_obj("new_data", cell_to_boc_b64(result.new_state.data).move_as_ok()); json_obj.leave(); @@ -422,7 +445,11 @@ const char *tvm_emulator_send_internal_message(void *tvm_emulator, const char *m } else { json_obj("missing_library", td::Bits256(result.missing_library).to_hex()); } - json_obj("actions", cell_to_boc_b64(result.actions).move_as_ok()); + if (result.actions.is_null()) { + json_obj("actions", td::JsonNull()); + } else { + json_obj("actions", cell_to_boc_b64(result.actions).move_as_ok()); + } json_obj("new_code", cell_to_boc_b64(result.new_state.code).move_as_ok()); json_obj("new_data", cell_to_boc_b64(result.new_state.data).move_as_ok()); json_obj.leave(); diff --git a/emulator/emulator-extern.h b/emulator/emulator-extern.h index ad5972de..db59a822 100644 --- a/emulator/emulator-extern.h +++ b/emulator/emulator-extern.h @@ -64,6 +64,14 @@ EMULATOR_EXPORT bool transaction_emulator_set_config(void *transaction_emulator, */ EMULATOR_EXPORT bool transaction_emulator_set_libs(void *transaction_emulator, const char* libs_boc); +/** + * @brief Enable or disable TVM debug primitives + * @param transaction_emulator Pointer to TransactionEmulator object + * @param debug_enabled Whether debug primitives should be enabled or not + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool transaction_emulator_set_debug_enabled(void *transaction_emulator, bool debug_enabled); + /** * @brief Emulate transaction * @param transaction_emulator Pointer to TransactionEmulator object @@ -134,6 +142,14 @@ EMULATOR_EXPORT bool tvm_emulator_set_c7(void *tvm_emulator, const char *address */ EMULATOR_EXPORT bool tvm_emulator_set_gas_limit(void *tvm_emulator, int64_t gas_limit); +/** + * @brief Enable or disable TVM debug primitives + * @param tvm_emulator Pointer to TVM emulator + * @param debug_enabled Whether debug primitives should be enabled or not + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool tvm_emulator_set_debug_enabled(void *tvm_emulator, bool debug_enabled); + /** * @brief Run get method * @param tvm_emulator Pointer to TVM emulator diff --git a/emulator/emulator_export_list b/emulator/emulator_export_list index b53a4114..64d4ff59 100644 --- a/emulator/emulator_export_list +++ b/emulator/emulator_export_list @@ -1,9 +1,11 @@ _transaction_emulator_create +_transaction_emulator_set_unixtime _transaction_emulator_set_lt _transaction_emulator_set_rand_seed _transaction_emulator_set_ignore_chksig _transaction_emulator_set_config _transaction_emulator_set_libs +_transaction_emulator_set_debug_enabled _transaction_emulator_emulate_transaction _transaction_emulator_destroy _emulator_set_verbosity_level @@ -11,6 +13,7 @@ _tvm_emulator_create _tvm_emulator_set_libraries _tvm_emulator_set_c7 _tvm_emulator_set_gas_limit +_tvm_emulator_set_debug_enabled _tvm_emulator_run_get_method _tvm_emulator_send_external_message _tvm_emulator_send_internal_message diff --git a/emulator/transaction-emulator.cpp b/emulator/transaction-emulator.cpp index 4a616418..7c8ade62 100644 --- a/emulator/transaction-emulator.cpp +++ b/emulator/transaction-emulator.cpp @@ -2,6 +2,7 @@ #include "transaction-emulator.h" #include "crypto/common/refcnt.hpp" #include "vm/cp0.h" +#include "tdutils/td/utils/Time.h" using td::Ref; using namespace std::string_literals; @@ -33,7 +34,7 @@ td::Result> TransactionEmu return fetch_res.move_as_error_prefix("cannot fetch config params "); } - vm::init_op_cp0(); + vm::init_op_cp0(debug_enabled_); if (!lt) { lt = lt_; @@ -47,9 +48,12 @@ td::Result> TransactionEmu compute_phase_cfg.with_vm_log = true; compute_phase_cfg.vm_log_verbosity = vm_log_verbosity_; + double start_time = td::Time::now(); auto res = create_transaction(msg_root, &account, utime, lt, trans_type, &storage_phase_cfg, &compute_phase_cfg, &action_phase_cfg); + double elapsed = td::Time::now() - start_time; + if(res.is_error()) { return res.move_as_error_prefix("cannot run message on account "); } @@ -58,7 +62,7 @@ td::Result> TransactionEmu if (!trans->compute_phase->accepted && trans->in_msg_extern) { auto vm_log = trans->compute_phase->vm_log; auto vm_exit_code = trans->compute_phase->exit_code; - return std::make_unique(std::move(vm_log), vm_exit_code); + return std::make_unique(std::move(vm_log), vm_exit_code, elapsed); } if (!trans->serialize()) { @@ -70,7 +74,8 @@ td::Result> TransactionEmu return td::Status::Error(PSLICE() << "cannot commit new transaction for smart contract"); } - return std::make_unique(std::move(trans_root), std::move(account), std::move(trans->compute_phase->vm_log), std::move(trans->compute_phase->actions)); + return std::make_unique(std::move(trans_root), std::move(account), + std::move(trans->compute_phase->vm_log), std::move(trans->compute_phase->actions), elapsed); } td::Result TransactionEmulator::emulate_transaction(block::Account&& account, td::Ref original_trans) { @@ -252,4 +257,8 @@ void TransactionEmulator::set_libs(vm::Dictionary &&libs) { libraries_ = std::forward(libs); } +void TransactionEmulator::set_debug_enabled(bool debug_enabled) { + debug_enabled_ = debug_enabled; +} + } // namespace emulator diff --git a/emulator/transaction-emulator.h b/emulator/transaction-emulator.h index 025bfa93..08343cb8 100644 --- a/emulator/transaction-emulator.h +++ b/emulator/transaction-emulator.h @@ -16,35 +16,37 @@ class TransactionEmulator { ton::LogicalTime lt_; td::BitArray<256> rand_seed_; bool ignore_chksig_; + bool debug_enabled_; public: TransactionEmulator(block::Config&& config, int vm_log_verbosity = 0) : config_(std::move(config)), libraries_(256), vm_log_verbosity_(vm_log_verbosity), - unixtime_(0), lt_(0), rand_seed_(td::BitArray<256>::zero()), ignore_chksig_(false) { + unixtime_(0), lt_(0), rand_seed_(td::BitArray<256>::zero()), ignore_chksig_(false), debug_enabled_(false) { } struct EmulationResult { std::string vm_log; + double elapsed_time; - EmulationResult(std::string vm_log_) : vm_log(vm_log_) {} + EmulationResult(std::string vm_log_, double elapsed_time_) : vm_log(vm_log_), elapsed_time(elapsed_time_) {} virtual ~EmulationResult() = default; }; struct EmulationSuccess: EmulationResult { td::Ref transaction; block::Account account; - td::Ref actions; + td::Ref actions; - EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, td::Ref actions_) : - EmulationResult(vm_log_), transaction(transaction_), account(account_) , actions(actions_) + EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, td::Ref actions_, double elapsed_time_) : + EmulationResult(vm_log_, elapsed_time_), transaction(transaction_), account(account_) , actions(actions_) {} }; struct EmulationExternalNotAccepted: EmulationResult { int vm_exit_code; - EmulationExternalNotAccepted(std::string vm_log_, int vm_exit_code_) : - EmulationResult(vm_log_), vm_exit_code(vm_exit_code_) + EmulationExternalNotAccepted(std::string vm_log_, int vm_exit_code_, double elapsed_time_) : + EmulationResult(vm_log_, elapsed_time_), vm_exit_code(vm_exit_code_) {} }; @@ -69,6 +71,7 @@ public: void set_ignore_chksig(bool ignore_chksig); void set_config(block::Config &&config); void set_libs(vm::Dictionary &&libs); + void set_debug_enabled(bool debug_enabled); private: bool check_state_update(const block::Account& account, const block::gen::Transaction::Record& trans); diff --git a/emulator/tvm-emulator.hpp b/emulator/tvm-emulator.hpp index dfbc20b1..0236f0ca 100644 --- a/emulator/tvm-emulator.hpp +++ b/emulator/tvm-emulator.hpp @@ -31,6 +31,10 @@ public: args_.set_config(config); } + void set_debug_enabled(bool debug_enabled) { + args_.set_debug_enabled(debug_enabled); + } + Answer run_get_method(int method_id, td::Ref stack) { return smc_.run_get_method(args_.set_stack(stack).set_method_id(method_id)); } diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 840dfd46..a5f36ff2 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -8,9 +8,11 @@ endif() set(STORAGE_SOURCE LoadSpeed.cpp MerkleTree.cpp + MicrochunkTree.cpp NodeActor.cpp PeerActor.cpp PeerState.cpp + SpeedLimiter.cpp Torrent.cpp TorrentCreator.cpp TorrentHeader.cpp @@ -25,13 +27,14 @@ set(STORAGE_SOURCE PartsHelper.h PeerActor.h PeerState.h + SpeedLimiter.h Torrent.h TorrentCreator.h TorrentHeader.h TorrentInfo.h TorrentMeta.h PeerManager.h - MicrochunkTree.h MicrochunkTree.cpp) + MicrochunkTree.h) set(STORAGE_CLI_SOURCE storage-cli.cpp ) diff --git a/storage/LoadSpeed.cpp b/storage/LoadSpeed.cpp index b47f4120..46aac2ff 100644 --- a/storage/LoadSpeed.cpp +++ b/storage/LoadSpeed.cpp @@ -31,13 +31,17 @@ double LoadSpeed::speed(td::Timestamp now) const { update(now); return (double)total_size_ / duration(now); } +void LoadSpeed::reset() { + events_ = {}; + total_size_ = 0; +} td::StringBuilder &operator<<(td::StringBuilder &sb, const LoadSpeed &speed) { return sb << td::format::as_size(static_cast(speed.speed())) << "/s"; } void LoadSpeed::update(td::Timestamp now) const { - while (duration(now) > 30) { + while (duration(now) > 10) { total_size_ -= events_.front().size; events_.pop(); } diff --git a/storage/LoadSpeed.h b/storage/LoadSpeed.h index 92d947f7..37115b88 100644 --- a/storage/LoadSpeed.h +++ b/storage/LoadSpeed.h @@ -28,6 +28,7 @@ class LoadSpeed { public: void add(td::uint64 size, td::Timestamp now = td::Timestamp::now()); double speed(td::Timestamp now = td::Timestamp::now()) const; + void reset(); friend td::StringBuilder &operator<<(td::StringBuilder &sb, const LoadSpeed &speed); private: diff --git a/storage/NodeActor.cpp b/storage/NodeActor.cpp index 041d74cd..e24fffff 100644 --- a/storage/NodeActor.cpp +++ b/storage/NodeActor.cpp @@ -25,27 +25,14 @@ #include "td/utils/Enumerator.h" #include "td/utils/tests.h" #include "td/utils/overloaded.h" -#include "tl-utils/common-utils.hpp" #include "tl-utils/tl-utils.hpp" #include "auto/tl/ton_api.hpp" #include "td/actor/MultiPromise.h" namespace ton { NodeActor::NodeActor(PeerId self_id, Torrent torrent, td::unique_ptr callback, - td::unique_ptr node_callback, std::shared_ptr db, bool should_download, - bool should_upload) - : self_id_(self_id) - , torrent_(std::move(torrent)) - , callback_(std::move(callback)) - , node_callback_(std::move(node_callback)) - , db_(std::move(db)) - , should_download_(should_download) - , should_upload_(should_upload) { -} - -NodeActor::NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr callback, - td::unique_ptr node_callback, std::shared_ptr db, bool should_download, - bool should_upload, DbInitialData db_initial_data) + td::unique_ptr node_callback, std::shared_ptr db, + SpeedLimiters speed_limiters, bool should_download, bool should_upload) : self_id_(self_id) , torrent_(std::move(torrent)) , callback_(std::move(callback)) @@ -53,6 +40,23 @@ NodeActor::NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr callback, + td::unique_ptr node_callback, std::shared_ptr db, + SpeedLimiters speed_limiters, bool should_download, bool should_upload, + DbInitialData db_initial_data) + : self_id_(self_id) + , torrent_(std::move(torrent)) + , callback_(std::move(callback)) + , node_callback_(std::move(node_callback)) + , db_(std::move(db)) + , should_download_(should_download) + , should_upload_(should_upload) + , added_at_(db_initial_data.added_at) + , speed_limiters_(std::move(speed_limiters)) , pending_set_file_priority_(std::move(db_initial_data.priorities)) , pieces_in_db_(std::move(db_initial_data.pieces_in_db)) { } @@ -100,12 +104,12 @@ void NodeActor::init_torrent() { } } - torrent_info_str_ = - std::make_shared(vm::std_boc_serialize(torrent_.get_info().as_cell()).move_as_ok()); + torrent_info_shared_ = std::make_shared(torrent_.get_info()); for (auto &p : peers_) { auto &state = p.second.state; - state->torrent_info_str_ = torrent_info_str_; + state->torrent_info_ = torrent_info_shared_; CHECK(!state->torrent_info_ready_.exchange(true)); + state->notify_peer(); } LOG(INFO) << "Inited torrent info for " << torrent_.get_hash().to_hex() << ": size=" << torrent_.get_info().file_size << ", pieces=" << torrent_.get_info().pieces_count(); @@ -185,7 +189,7 @@ void NodeActor::loop_will_upload() { auto &state = it.second.state; bool needed = false; if (state->peer_state_ready_) { - needed = state->peer_state_.load().want_download; + needed = state->peer_state_.load().want_download && state->peer_online_; } peers.emplace_back(!needed, !state->node_state_.load().want_download, -it.second.download_speed.speed(), it.first); } @@ -247,6 +251,10 @@ void NodeActor::loop() { } wait_for_completion_.clear(); is_completed_ = true; + download_speed_.reset(); + for (auto &peer : peers_) { + peer.second.download_speed.reset(); + } callback_->on_completed(); } } @@ -398,6 +406,12 @@ void NodeActor::set_should_download(bool should_download) { return; } should_download_ = should_download; + if (!should_download_) { + download_speed_.reset(); + for (auto &peer : peers_) { + peer.second.download_speed.reset(); + } + } db_store_torrent(); yield(); } @@ -407,7 +421,14 @@ void NodeActor::set_should_upload(bool should_upload) { return; } should_upload_ = should_upload; + if (!should_upload_) { + upload_speed_.reset(); + for (auto &peer : peers_) { + peer.second.upload_speed.reset(); + } + } db_store_torrent(); + will_upload_at_ = td::Timestamp::now(); yield(); } @@ -478,6 +499,7 @@ void NodeActor::loop_start_stop_peers() { if (peer.actor.empty()) { auto &state = peer.state = std::make_shared(peer.notifier.get()); + state->speed_limiters_ = speed_limiters_; if (torrent_.inited_info()) { std::vector node_ready_parts; for (td::uint32 i = 0; i < parts_.parts.size(); i++) { @@ -486,7 +508,7 @@ void NodeActor::loop_start_stop_peers() { } } state->node_ready_parts_.add_elements(std::move(node_ready_parts)); - state->torrent_info_str_ = torrent_info_str_; + state->torrent_info_ = torrent_info_shared_; state->torrent_info_ready_ = true; } else { state->torrent_info_response_callback_ = [SelfId = actor_id(this)](td::BufferSlice data) { @@ -575,6 +597,11 @@ void NodeActor::loop_peer(const PeerId &peer_id, Peer &peer) { for (auto part_id : state->peer_ready_parts_.read()) { parts_helper_.on_peer_part_ready(peer.peer_token, part_id); } + if (state->peer_online_set_.load()) { + state->peer_online_set_ = false; + will_upload_at_ = td::Timestamp::now(); + loop_will_upload(); + } // Answer queries from peer bool should_notify_peer = false; @@ -600,7 +627,7 @@ void NodeActor::loop_peer(const PeerId &peer_id, Peer &peer) { TRY_RESULT(proof_serialized, vm::std_boc_serialize(std::move(proof))); res.proof = std::move(proof_serialized); res.data = td::BufferSlice(std::move(data)); - td::uint64 size = res.data.size() + res.proof.size(); + td::uint64 size = res.data.size(); upload_speed_.add(size); peer.upload_speed.add(size); return std::move(res); @@ -701,10 +728,11 @@ void NodeActor::db_store_torrent() { if (!db_) { return; } - auto obj = create_tl_object(); + auto obj = create_tl_object(); obj->active_download_ = should_download_; obj->active_upload_ = should_upload_; obj->root_dir_ = torrent_.get_root_dir(); + obj->added_at_ = added_at_; db_->set(create_hash_tl_object(torrent_.get_hash()), serialize_tl_object(obj, true), [](td::Result R) { if (R.is_error()) { @@ -823,16 +851,18 @@ void NodeActor::db_update_pieces_list() { } void NodeActor::load_from_db(std::shared_ptr db, td::Bits256 hash, td::unique_ptr callback, - td::unique_ptr node_callback, + td::unique_ptr node_callback, SpeedLimiters speed_limiters, td::Promise> promise) { class Loader : public td::actor::Actor { public: Loader(std::shared_ptr db, td::Bits256 hash, td::unique_ptr callback, - td::unique_ptr node_callback, td::Promise> promise) + td::unique_ptr node_callback, SpeedLimiters speed_limiters, + td::Promise> promise) : db_(std::move(db)) , hash_(hash) , callback_(std::move(callback)) , node_callback_(std::move(node_callback)) + , speed_limiters_(std::move(speed_limiters)) , promise_(std::move(promise)) { } @@ -842,9 +872,9 @@ void NodeActor::load_from_db(std::shared_ptr db, td::Bits256 hash, t } void start_up() override { - db::db_get( + db::db_get( *db_, create_hash_tl_object(hash_), false, - [SelfId = actor_id(this)](td::Result> R) { + [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &Loader::finish, R.move_as_error_prefix("Torrent: ")); } else { @@ -853,10 +883,20 @@ void NodeActor::load_from_db(std::shared_ptr db, td::Bits256 hash, t }); } - void got_torrent(tl_object_ptr obj) { - root_dir_ = std::move(obj->root_dir_); - active_download_ = obj->active_download_; - active_upload_ = obj->active_upload_; + void got_torrent(tl_object_ptr obj) { + ton_api::downcast_call(*obj, td::overloaded( + [&](ton_api::storage_db_torrent &t) { + root_dir_ = std::move(t.root_dir_); + active_download_ = t.active_download_; + active_upload_ = t.active_upload_; + added_at_ = (td::uint32)td::Clocks::system(); + }, + [&](ton_api::storage_db_torrentV2 &t) { + root_dir_ = std::move(t.root_dir_); + active_download_ = t.active_download_; + active_upload_ = t.active_upload_; + added_at_ = t.added_at_; + })); db_->get(create_hash_tl_object(hash_), [SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -980,9 +1020,10 @@ void NodeActor::load_from_db(std::shared_ptr db, td::Bits256 hash, t DbInitialData data; data.priorities = std::move(priorities_); data.pieces_in_db = std::move(pieces_in_db_); + data.added_at = added_at_; finish(td::actor::create_actor("Node", 1, torrent_.unwrap(), std::move(callback_), - std::move(node_callback_), std::move(db_), active_download_, - active_upload_, std::move(data))); + std::move(node_callback_), std::move(db_), std::move(speed_limiters_), + active_download_, active_upload_, std::move(data))); } private: @@ -990,18 +1031,20 @@ void NodeActor::load_from_db(std::shared_ptr db, td::Bits256 hash, t td::Bits256 hash_; td::unique_ptr callback_; td::unique_ptr node_callback_; + SpeedLimiters speed_limiters_; td::Promise> promise_; std::string root_dir_; bool active_download_{false}; bool active_upload_{false}; + td::uint32 added_at_; td::optional torrent_; std::vector priorities_; std::set pieces_in_db_; size_t remaining_pieces_in_db_ = 0; }; td::actor::create_actor("loader", std::move(db), hash, std::move(callback), std::move(node_callback), - std::move(promise)) + std::move(speed_limiters), std::move(promise)) .release(); } diff --git a/storage/NodeActor.h b/storage/NodeActor.h index f8cadce2..2f0a3232 100644 --- a/storage/NodeActor.h +++ b/storage/NodeActor.h @@ -23,6 +23,7 @@ #include "PartsHelper.h" #include "PeerActor.h" #include "Torrent.h" +#include "SpeedLimiter.h" #include "td/utils/Random.h" #include "td/utils/Variant.h" @@ -31,6 +32,7 @@ #include "db.h" namespace ton { + class NodeActor : public td::actor::Actor { public: class NodeCallback { @@ -60,14 +62,15 @@ class NodeActor : public td::actor::Actor { struct DbInitialData { std::vector priorities; std::set pieces_in_db; + td::uint32 added_at; }; NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr callback, - td::unique_ptr node_callback, std::shared_ptr db, bool should_download = true, - bool should_upload = true); + td::unique_ptr node_callback, std::shared_ptr db, SpeedLimiters speed_limiters, + bool should_download = true, bool should_upload = true); NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr callback, - td::unique_ptr node_callback, std::shared_ptr db, bool should_download, - bool should_upload, DbInitialData db_initial_data); + td::unique_ptr node_callback, std::shared_ptr db, SpeedLimiters speed_limiters, + bool should_download, bool should_upload, DbInitialData db_initial_data); void start_peer(PeerId peer_id, td::Promise> promise); struct NodeState { @@ -76,11 +79,12 @@ class NodeActor : public td::actor::Actor { bool active_upload; double download_speed; double upload_speed; + td::uint32 added_at; const std::vector &file_priority; }; void with_torrent(td::Promise promise) { promise.set_value(NodeState{torrent_, should_download_, should_upload_, download_speed_.speed(), - upload_speed_.speed(), file_priority_}); + upload_speed_.speed(), added_at_, file_priority_}); } std::string get_stats_str(); @@ -98,20 +102,22 @@ class NodeActor : public td::actor::Actor { void get_peers_info(td::Promise> promise); static void load_from_db(std::shared_ptr db, td::Bits256 hash, td::unique_ptr callback, - td::unique_ptr node_callback, + td::unique_ptr node_callback, SpeedLimiters speed_limiters, td::Promise> promise); static void cleanup_db(std::shared_ptr db, td::Bits256 hash, td::Promise promise); private: PeerId self_id_; ton::Torrent torrent_; - std::shared_ptr torrent_info_str_; + std::shared_ptr torrent_info_shared_; std::vector file_priority_; td::unique_ptr callback_; td::unique_ptr node_callback_; std::shared_ptr db_; bool should_download_{false}; bool should_upload_{false}; + td::uint32 added_at_{0}; + SpeedLimiters speed_limiters_; class Notifier : public td::actor::Actor { public: diff --git a/storage/PeerActor.cpp b/storage/PeerActor.cpp index b2bf5a89..0cb21c0a 100644 --- a/storage/PeerActor.cpp +++ b/storage/PeerActor.cpp @@ -24,6 +24,7 @@ #include "td/utils/overloaded.h" #include "td/utils/Random.h" +#include "vm/boc.h" namespace ton { @@ -44,7 +45,7 @@ PeerActor::PeerActor(td::unique_ptr callback, std::shared_ptr -td::uint64 PeerActor::create_and_send_query(ArgsT &&... args) { +td::uint64 PeerActor::create_and_send_query(ArgsT &&...args) { return send_query(ton::create_serialize_tl_object(std::forward(args)...)); } @@ -86,7 +87,9 @@ void PeerActor::on_ping_result(td::Result r_answer) { void PeerActor::on_pong() { wait_pong_till_ = td::Timestamp::in(10); - state_->peer_online_ = true; + if (!state_->peer_online_.exchange(true)) { + state_->peer_online_set_ = true; + } notify_node(); } @@ -115,6 +118,11 @@ void PeerActor::on_get_piece_result(PartId piece_id, td::Result res.proof = std::move(piece->proof_); return std::move(res); }(); + if (res.is_error()) { + LOG(DEBUG) << "getPiece " << piece_id << "query: " << res.error(); + } else { + LOG(DEBUG) << "getPiece " << piece_id << "query: OK"; + } state_->node_queries_results_.add_element(std::make_pair(piece_id, std::move(res))); notify_node(); } @@ -130,13 +138,16 @@ void PeerActor::on_get_info_result(td::Result r_answer) { next_get_info_at_ = td::Timestamp::in(5.0); alarm_timestamp().relax(next_get_info_at_); if (r_answer.is_error()) { + LOG(DEBUG) << "getTorrentInfo query: " << r_answer.move_as_error(); return; } auto R = fetch_tl_object(r_answer.move_as_ok(), true); if (R.is_error()) { + LOG(DEBUG) << "getTorrentInfo query: " << R.move_as_error(); return; } td::BufferSlice data = std::move(R.ok_ref()->data_); + LOG(DEBUG) << "getTorrentInfo query: got result (" << data.size() << " bytes)"; if (!data.empty() && !state_->torrent_info_ready_) { state_->torrent_info_response_callback_(std::move(data)); } @@ -245,6 +256,8 @@ void PeerActor::loop_update_init() { update_state_query_.state = node_state; update_state_query_.query_id = 0; + LOG(DEBUG) << "Sending updateInit query (" << update_state_query_.state.want_download << ", " + << update_state_query_.state.will_upload << ", offset=" << peer_init_offset_ * 8 << ")"; update_query_id_ = send_query(std::move(query)); } @@ -265,13 +278,15 @@ void PeerActor::loop_update_state() { auto query = create_update_query( ton::create_tl_object(to_ton_api(update_state_query_.state))); + LOG(DEBUG) << "Sending updateState query (" << update_state_query_.state.want_download << ", " + << update_state_query_.state.will_upload << ")"; update_state_query_.query_id = send_query(std::move(query)); } void PeerActor::update_have_pieces() { auto node_ready_parts = state_->node_ready_parts_.read(); for (auto piece_id : node_ready_parts) { - if (piece_id < peer_init_offset_ + UPDATE_INIT_BLOCK_SIZE) { + if (piece_id < (peer_init_offset_ + UPDATE_INIT_BLOCK_SIZE) * 8 && !have_pieces_.get(piece_id)) { have_pieces_list_.push_back(piece_id); } have_pieces_.set_one(piece_id); @@ -291,23 +306,49 @@ void PeerActor::loop_update_pieces() { have_pieces_list_.erase(have_pieces_list_.end() - count, have_pieces_list_.end()); auto query = create_update_query(ton::create_tl_object( td::transform(sent_have_pieces_list_, [](auto x) { return static_cast(x); }))); + LOG(DEBUG) << "Sending updateHavePieces query (" << sent_have_pieces_list_.size() << " pieces)"; update_query_id_ = send_query(std::move(query)); } } void PeerActor::loop_get_torrent_info() { - if (get_info_query_id_ || state_->torrent_info_ready_) { + if (state_->torrent_info_ready_) { + if (!torrent_info_) { + torrent_info_ = state_->torrent_info_; + for (const auto &u : pending_update_peer_parts_) { + process_update_peer_parts(u); + } + pending_update_peer_parts_.clear(); + } + return; + } + if (get_info_query_id_) { return; } if (next_get_info_at_ && !next_get_info_at_.is_in_past()) { return; } + LOG(DEBUG) << "Sending getTorrentInfo query"; get_info_query_id_ = create_and_send_query(); } void PeerActor::loop_node_get_piece() { for (auto part : state_->node_queries_.read()) { - node_get_piece_.emplace(part, NodePieceQuery{}); + if (state_->speed_limiters_.download.empty()) { + node_get_piece_.emplace(part, NodePieceQuery{}); + } else { + if (!torrent_info_) { + CHECK(state_->torrent_info_ready_); + loop_get_torrent_info(); + } + auto piece_size = + std::min(torrent_info_->piece_size, torrent_info_->file_size - part * torrent_info_->piece_size); + td::actor::send_closure(state_->speed_limiters_.download, &SpeedLimiter::enqueue, (double)piece_size, + td::Timestamp::in(3.0), [part, SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &PeerActor::node_get_piece_query_ready, part, + std::move(R)); + }); + } } for (auto &query_it : node_get_piece_) { @@ -315,11 +356,21 @@ void PeerActor::loop_node_get_piece() { continue; } + LOG(DEBUG) << "Sending getPiece " << query_it.first << " query"; query_it.second.query_id = create_and_send_query(static_cast(query_it.first)); } } +void PeerActor::node_get_piece_query_ready(PartId part, td::Result R) { + if (R.is_error()) { + on_get_piece_result(part, R.move_as_error()); + } else { + node_get_piece_.emplace(part, NodePieceQuery{}); + } + schedule_loop(); +} + void PeerActor::loop_peer_get_piece() { // process answers for (auto &p : state_->peer_queries_results_.read()) { @@ -328,9 +379,26 @@ void PeerActor::loop_peer_get_piece() { if (promise_it == peer_get_piece_.end()) { continue; } - promise_it->second.promise.set_result(p.second.move_map([](PeerState::Part part) { - return ton::create_serialize_tl_object(std::move(part.proof), std::move(part.data)); - })); + td::Promise promise = + [i = p.first, promise = std::move(promise_it->second.promise)](td::Result R) mutable { + LOG(DEBUG) << "Responding to getPiece " << i << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + promise.set_result(R.move_map([](PeerState::Part part) { + return create_serialize_tl_object(std::move(part.proof), std::move(part.data)); + })); + }; + if (p.second.is_error()) { + promise.set_error(p.second.move_as_error()); + } else { + auto part = p.second.move_as_ok(); + auto size = (double)part.data.size(); + td::Promise P = promise.wrap([part = std::move(part)](td::Unit) mutable { return std::move(part); }); + if (state_->speed_limiters_.upload.empty()) { + P.set_result(td::Unit()); + } else { + td::actor::send_closure(state_->speed_limiters_.upload, &SpeedLimiter::enqueue, size, td::Timestamp::in(3.0), + std::move(P)); + } + } peer_get_piece_.erase(promise_it); notify_node(); } @@ -373,18 +441,8 @@ void PeerActor::execute_add_update(ton::ton_api::storage_addUpdate &add_update, promise.set_error(td::Status::Error(404, "INVALID_SESSION")); return; } - promise.set_value(ton::create_serialize_tl_object()); - std::vector new_peer_ready_parts; - auto add_piece = [&](PartId id) { - if (!peer_have_pieces_.get(id)) { - peer_have_pieces_.set_one(id); - new_peer_ready_parts.push_back(id); - notify_node(); - } - }; - auto seqno = static_cast(add_update.seqno_); auto update_peer_state = [&](PeerState::State peer_state) { if (peer_seqno_ >= seqno) { @@ -393,22 +451,48 @@ void PeerActor::execute_add_update(ton::ton_api::storage_addUpdate &add_update, if (state_->peer_state_ready_ && state_->peer_state_.load() == peer_state) { return; } + LOG(DEBUG) << "Processing update peer state query (" << peer_state.want_download << ", " << peer_state.will_upload + << ")"; peer_seqno_ = seqno; state_->peer_state_.exchange(peer_state); state_->peer_state_ready_ = true; notify_node(); }; + downcast_call( + *add_update.update_, + td::overloaded( + [&](const ton::ton_api::storage_updateHavePieces &have_pieces) {}, + [&](const ton::ton_api::storage_updateState &state) { update_peer_state(from_ton_api(*state.state_)); }, + [&](const ton::ton_api::storage_updateInit &init) { update_peer_state(from_ton_api(*init.state_)); })); + if (torrent_info_) { + process_update_peer_parts(add_update.update_); + } else { + pending_update_peer_parts_.push_back(std::move(add_update.update_)); + } +} - downcast_call(*add_update.update_, +void PeerActor::process_update_peer_parts(const tl_object_ptr &update) { + CHECK(torrent_info_); + td::uint64 pieces_count = torrent_info_->pieces_count(); + std::vector new_peer_ready_parts; + auto add_piece = [&](PartId id) { + if (id < pieces_count && !peer_have_pieces_.get(id)) { + peer_have_pieces_.set_one(id); + new_peer_ready_parts.push_back(id); + notify_node(); + } + }; + downcast_call(*update, td::overloaded( - [&](ton::ton_api::storage_updateHavePieces &have_pieces) { + [&](const ton::ton_api::storage_updateHavePieces &have_pieces) { + LOG(DEBUG) << "Processing updateHavePieces query (" << have_pieces.piece_id_ << " pieces)"; for (auto id : have_pieces.piece_id_) { add_piece(id); } }, - [&](ton::ton_api::storage_updateState &state) { update_peer_state(from_ton_api(*state.state_)); }, - [&](ton::ton_api::storage_updateInit &init) { - update_peer_state(from_ton_api(*init.state_)); + [&](const ton::ton_api::storage_updateState &state) {}, + [&](const ton::ton_api::storage_updateInit &init) { + LOG(DEBUG) << "Processing updateInit query (offset=" << init.have_pieces_offset_ * 8 << ")"; td::Bitset new_bitset; new_bitset.set_raw(init.have_pieces_.as_slice().str()); size_t offset = init.have_pieces_offset_ * 8; @@ -428,7 +512,13 @@ void PeerActor::execute_get_piece(ton::ton_api::storage_getPiece &get_piece, td: void PeerActor::execute_get_torrent_info(td::Promise promise) { td::BufferSlice result = create_serialize_tl_object( - state_->torrent_info_ready_ ? state_->torrent_info_str_->clone() : td::BufferSlice()); + state_->torrent_info_ready_ ? vm::std_boc_serialize(state_->torrent_info_->as_cell()).move_as_ok() + : td::BufferSlice()); + if (state_->torrent_info_ready_) { + LOG(DEBUG) << "Responding to getTorrentInfo: " << result.size() << " bytes"; + } else { + LOG(DEBUG) << "Responding to getTorrentInfo: no info"; + } promise.set_result(std::move(result)); } } // namespace ton diff --git a/storage/PeerActor.h b/storage/PeerActor.h index fd34bc88..a4229e16 100644 --- a/storage/PeerActor.h +++ b/storage/PeerActor.h @@ -59,6 +59,10 @@ class PeerActor : public td::actor::Actor { // startSession td::uint64 node_session_id_; td::Bitset peer_have_pieces_; + std::shared_ptr torrent_info_; + std::vector> pending_update_peer_parts_; + + void process_update_peer_parts(const tl_object_ptr &update); // update td::optional peer_session_id_; @@ -112,6 +116,7 @@ class PeerActor : public td::actor::Actor { td::BufferSlice create_update_query(ton::tl_object_ptr update); void loop_node_get_piece(); + void node_get_piece_query_ready(PartId part, td::Result R); void loop_peer_get_piece(); diff --git a/storage/PeerState.h b/storage/PeerState.h index c578b544..29c080a2 100644 --- a/storage/PeerState.h +++ b/storage/PeerState.h @@ -23,6 +23,8 @@ #include "td/utils/optional.h" #include "td/actor/actor.h" +#include "TorrentInfo.h" +#include "SpeedLimiter.h" #include #include @@ -100,6 +102,8 @@ struct PeerState { std::atomic_bool peer_state_ready_{false}; std::atomic peer_state_{State{false, false}}; std::atomic_bool peer_online_{false}; + std::atomic_bool peer_online_set_{false}; + SpeedLimiters speed_limiters_; struct Part { td::BufferSlice proof; @@ -121,7 +125,7 @@ struct PeerState { // Node -> Peer std::atomic_bool torrent_info_ready_{false}; - std::shared_ptr torrent_info_str_; + std::shared_ptr torrent_info_; std::function torrent_info_response_callback_; const td::actor::ActorId<> node; diff --git a/storage/SpeedLimiter.cpp b/storage/SpeedLimiter.cpp new file mode 100644 index 00000000..952005fe --- /dev/null +++ b/storage/SpeedLimiter.cpp @@ -0,0 +1,82 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "SpeedLimiter.h" + +namespace ton { + +SpeedLimiter::SpeedLimiter(double max_speed) : max_speed_(max_speed) { +} + +void SpeedLimiter::set_max_speed(double max_speed) { + max_speed_ = max_speed; + auto old_queue = std::move(queue_); + unlock_at_ = (queue_.empty() ? td::Timestamp::now() : queue_.front().execute_at_); + queue_ = {}; + while (!old_queue.empty()) { + auto &e = old_queue.front(); + enqueue(e.size_, e.timeout_, std::move(e.promise_)); + old_queue.pop(); + } + process_queue(); +} + +void SpeedLimiter::enqueue(double size, td::Timestamp timeout, td::Promise promise) { + if (max_speed_ < 0.0) { + promise.set_result(td::Unit()); + return; + } + if (max_speed_ == 0.0) { + promise.set_error(td::Status::Error("Speed limit is 0")); + return; + } + if (timeout < unlock_at_) { + promise.set_error(td::Status::Error("Timeout caused by speed limit")); + return; + } + if (queue_.empty() && unlock_at_.is_in_past()) { + unlock_at_ = td::Timestamp::now(); + promise.set_result(td::Unit()); + } else { + queue_.push({unlock_at_, size, timeout, std::move(promise)}); + } + unlock_at_ = td::Timestamp::in(size / max_speed_, unlock_at_); + if (!queue_.empty()) { + alarm_timestamp() = queue_.front().execute_at_; + } +} + +void SpeedLimiter::alarm() { + process_queue(); +} + +void SpeedLimiter::process_queue() { + while (!queue_.empty()) { + auto &e = queue_.front(); + if (e.execute_at_.is_in_past()) { + e.promise_.set_result(td::Unit()); + queue_.pop(); + } else { + break; + } + } + if (!queue_.empty()) { + alarm_timestamp() = queue_.front().execute_at_; + } +} + +} // namespace ton diff --git a/storage/SpeedLimiter.h b/storage/SpeedLimiter.h new file mode 100644 index 00000000..b6230732 --- /dev/null +++ b/storage/SpeedLimiter.h @@ -0,0 +1,52 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#pragma once +#include "td/actor/actor.h" +#include + +namespace ton { + +class SpeedLimiter : public td::actor::Actor { + public: + explicit SpeedLimiter(double max_speed); + + void set_max_speed(double max_speed); // Negative = unlimited + void enqueue(double size, td::Timestamp timeout, td::Promise promise); + + void alarm() override; + + private: + double max_speed_ = -1.0; + td::Timestamp unlock_at_ = td::Timestamp::never(); + + struct Event { + td::Timestamp execute_at_; + double size_; + td::Timestamp timeout_; + td::Promise promise_; + }; + std::queue queue_; + + void process_queue(); +}; + +struct SpeedLimiters { + td::actor::ActorId download, upload; +}; + +} // namespace ton diff --git a/storage/Torrent.cpp b/storage/Torrent.cpp index 1dafb999..6807519c 100644 --- a/storage/Torrent.cpp +++ b/storage/Torrent.cpp @@ -266,7 +266,9 @@ td::Result Torrent::get_piece_data(td::uint64 piece_i) { if (!inited_info_) { return td::Status::Error("Torrent info not inited"); } - CHECK(piece_i < info_.pieces_count()); + if (piece_i >= info_.pieces_count()) { + return td::Status::Error("Piece idx is too big"); + } if (!piece_is_ready_[piece_i]) { return td::Status::Error("Piece is not ready"); } @@ -291,7 +293,9 @@ td::Result> Torrent::get_piece_proof(td::uint64 piece_i) { if (!inited_info_) { return td::Status::Error("Torrent info not inited"); } - CHECK(piece_i < info_.pieces_count()); + if (piece_i >= info_.pieces_count()) { + return td::Status::Error("Piece idx is too big"); + } return merkle_tree_.gen_proof(piece_i, piece_i); } @@ -305,7 +309,9 @@ td::Status Torrent::add_piece(td::uint64 piece_i, td::Slice data, td::Ref= info_.pieces_count()) { + return td::Status::Error("Piece idx is too big"); + } if (piece_is_ready_[piece_i]) { return td::Status::OK(); } diff --git a/storage/TorrentCreator.cpp b/storage/TorrentCreator.cpp index 35fb8212..ab060023 100644 --- a/storage/TorrentCreator.cpp +++ b/storage/TorrentCreator.cpp @@ -29,18 +29,22 @@ #include "TorrentHeader.hpp" namespace ton { +static bool is_dir_slash(char c) { + return (c == TD_DIR_SLASH) | (c == '/'); +} + td::Result Torrent::Creator::create_from_path(Options options, td::CSlice raw_path) { TRY_RESULT(path, td::realpath(raw_path)); TRY_RESULT(stat, td::stat(path)); std::string root_dir = path; - while (!root_dir.empty() && root_dir.back() == TD_DIR_SLASH) { + while (!root_dir.empty() && is_dir_slash(root_dir.back())) { root_dir.pop_back(); } - while (!root_dir.empty() && root_dir.back() != TD_DIR_SLASH) { + while (!root_dir.empty() && !is_dir_slash(root_dir.back())) { root_dir.pop_back(); } if (stat.is_dir_) { - if (!path.empty() && path.back() != TD_DIR_SLASH) { + if (!path.empty() && !is_dir_slash(path.back())) { path += TD_DIR_SLASH; } if (!options.dir_name) { @@ -50,7 +54,20 @@ td::Result Torrent::Creator::create_from_path(Options options, td::CSli td::Status status; auto walk_status = td::WalkPath::run(path, [&](td::CSlice name, td::WalkPath::Type type) { if (type == td::WalkPath::Type::NotDir) { - status = creator.add_file(td::PathView::relative(name, path), name); + std::string rel_name = td::PathView::relative(name, path).str(); + td::Slice file_name = rel_name; + for (size_t i = 0; i < rel_name.size(); ++i) { + if (is_dir_slash(rel_name[i])) { + rel_name[i] = '/'; + file_name = td::Slice(rel_name.c_str() + i + 1, rel_name.c_str() + rel_name.size()); + } + } + // Exclude OS-created files that can be modified automatically and thus break some torrent pieces + if (file_name == ".DS_Store" || td::to_lower(file_name) == "desktop.ini" || + td::to_lower(file_name) == "thumbs.db") { + return td::WalkPath::Action::Continue; + } + status = creator.add_file(rel_name, name); if (status.is_error()) { return td::WalkPath::Action::Abort; } diff --git a/storage/TorrentInfo.cpp b/storage/TorrentInfo.cpp index 48b0d592..26aa092e 100644 --- a/storage/TorrentInfo.cpp +++ b/storage/TorrentInfo.cpp @@ -73,6 +73,12 @@ td::Status TorrentInfo::validate() const { if (description.size() > 1024) { return td::Status::Error("Description is too long"); } + if (piece_size > (1 << 23)) { + return td::Status::Error("Piece size is too big"); + } + if (pieces_count() >= (1ULL << 31)) { + return td::Status::Error("Too many pieces"); + } return td::Status::OK(); } } // namespace ton diff --git a/storage/storage-cli.cpp b/storage/storage-cli.cpp index 43ddbeaf..858433d1 100644 --- a/storage/storage-cli.cpp +++ b/storage/storage-cli.cpp @@ -458,9 +458,9 @@ class StorageCli : public td::actor::Actor { } auto callback = td::make_unique(actor_id(this), ptr->id, std::move(on_completed)); auto context = PeerManager::create_callback(ptr->peer_manager.get()); - ptr->node = - td::actor::create_actor(PSLICE() << "Node#" << self_id, self_id, ptr->torrent.unwrap(), - std::move(callback), std::move(context), nullptr, should_download); + ptr->node = td::actor::create_actor(PSLICE() << "Node#" << self_id, self_id, ptr->torrent.unwrap(), + std::move(callback), std::move(context), nullptr, + ton::SpeedLimiters{}, should_download); td::TerminalIO::out() << "Torrent #" << ptr->id << " started\n"; promise.release().release(); if (promise) { diff --git a/storage/storage-daemon/StorageManager.cpp b/storage/storage-daemon/StorageManager.cpp index 8edd5224..cf93f735 100644 --- a/storage/storage-daemon/StorageManager.cpp +++ b/storage/storage-daemon/StorageManager.cpp @@ -50,6 +50,29 @@ void StorageManager::start_up() { db_ = std::make_shared( std::make_shared(td::RocksDb::open(db_root_ + "/torrent-db").move_as_ok())); + + db::db_get( + *db_, create_hash_tl_object(), true, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + LOG(ERROR) << "Failed to load config from db: " << R.move_as_error(); + td::actor::send_closure(SelfId, &StorageManager::loaded_config_from_db, nullptr); + } else { + td::actor::send_closure(SelfId, &StorageManager::loaded_config_from_db, R.move_as_ok()); + } + }); +} + +void StorageManager::loaded_config_from_db(tl_object_ptr config) { + if (config) { + LOG(INFO) << "Loaded config from DB. Speed limits: download=" << config->download_speed_limit_ + << ", upload=" << config->upload_speed_limit_; + download_speed_limit_ = config->download_speed_limit_; + upload_speed_limit_ = config->upload_speed_limit_; + td::actor::send_closure(download_speed_limiter_, &SpeedLimiter::set_max_speed, download_speed_limit_); + td::actor::send_closure(upload_speed_limiter_, &SpeedLimiter::set_max_speed, upload_speed_limit_); + } + db::db_get( *db_, create_hash_tl_object(), true, [SelfId = actor_id(this)](td::Result> R) { @@ -79,6 +102,7 @@ void StorageManager::load_torrents_from_db(std::vector torrents) { client_mode_, overlays_, adnl_, rldp_); NodeActor::load_from_db( db_, hash, create_callback(hash, entry.closing_state), PeerManager::create_callback(entry.peer_manager.get()), + SpeedLimiters{download_speed_limiter_.get(), upload_speed_limiter_.get()}, [SelfId = actor_id(this), hash, promise = ig.get_promise()](td::Result> R) mutable { td::actor::send_closure(SelfId, &StorageManager::loaded_torrent_from_db, hash, std::move(R)); @@ -162,9 +186,9 @@ td::Status StorageManager::add_torrent_impl(Torrent torrent, bool start_download client_mode_, overlays_, adnl_, rldp_); auto context = PeerManager::create_callback(entry.peer_manager.get()); LOG(INFO) << "Added torrent " << hash.to_hex() << " , root_dir = " << torrent.get_root_dir(); - entry.actor = - td::actor::create_actor("Node", 1, std::move(torrent), create_callback(hash, entry.closing_state), - std::move(context), db_, start_download, allow_upload); + entry.actor = td::actor::create_actor( + "Node", 1, std::move(torrent), create_callback(hash, entry.closing_state), std::move(context), db_, + SpeedLimiters{download_speed_limiter_.get(), upload_speed_limiter_.get()}, start_download, allow_upload); return td::Status::OK(); } @@ -210,6 +234,18 @@ void StorageManager::get_all_torrents(td::Promise> prom promise.set_result(std::move(result)); } +void StorageManager::db_store_config() { + auto config = create_tl_object(); + config->download_speed_limit_ = download_speed_limit_; + config->upload_speed_limit_ = upload_speed_limit_; + db_->set(create_hash_tl_object(), serialize_tl_object(config, true), + [](td::Result R) { + if (R.is_error()) { + LOG(ERROR) << "Failed to save config to db: " << R.move_as_error(); + } + }); +} + void StorageManager::db_store_torrent_list() { std::vector torrents; for (const auto& p : torrents_) { @@ -259,19 +295,55 @@ void StorageManager::load_from(td::Bits256 hash, td::optional meta, std::move(promise)); } +static bool try_rm_empty_dir(const std::string& path) { + auto stat = td::stat(path); + if (stat.is_error() || !stat.ok().is_dir_) { + return true; + } + size_t cnt = 0; + td::WalkPath::run(path, [&](td::CSlice name, td::WalkPath::Type type) { + if (type != td::WalkPath::Type::ExitDir) { + ++cnt; + } + if (cnt < 2) { + return td::WalkPath::Action::Continue; + } else { + return td::WalkPath::Action::Abort; + } + }).ignore(); + if (cnt == 1) { + td::rmdir(path).ignore(); + return true; + } + return false; +} + void StorageManager::on_torrent_closed(Torrent torrent, std::shared_ptr closing_state) { if (!closing_state->removing) { return; } if (closing_state->remove_files && torrent.inited_header()) { + // Ignore all errors: files may just not exist size_t files_count = torrent.get_files_count().unwrap(); for (size_t i = 0; i < files_count; ++i) { std::string path = torrent.get_file_path(i); td::unlink(path).ignore(); - // TODO: Check errors, remove empty directories + std::string name = torrent.get_file_name(i).str(); + for (int j = (int)name.size() - 1; j >= 0; --j) { + if (name[j] == '/') { + name.resize(j + 1); + if (!try_rm_empty_dir(torrent.get_root_dir() + '/' + torrent.get_header().dir_name + '/' + name)) { + break; + } + } + } + if (!torrent.get_header().dir_name.empty()) { + try_rm_empty_dir(torrent.get_root_dir() + '/' + torrent.get_header().dir_name); + } } } - td::rmrf(db_root_ + "/torrent-files/" + torrent.get_hash().to_hex()).ignore(); + std::string path = db_root_ + "/torrent-files/" + torrent.get_hash().to_hex(); + td::rmrf(path).ignore(); NodeActor::cleanup_db(db_, torrent.get_hash(), [promise = std::move(closing_state->promise)](td::Result R) mutable { if (R.is_error()) { @@ -292,4 +364,28 @@ void StorageManager::get_peers_info(td::Bits256 hash, td::actor::send_closure(entry->actor, &NodeActor::get_peers_info, std::move(promise)); } +void StorageManager::get_speed_limits(td::Promise> promise) { + promise.set_result(std::make_pair(download_speed_limit_, upload_speed_limit_)); +} + +void StorageManager::set_download_speed_limit(double max_speed) { + if (max_speed < 0.0) { + max_speed = -1.0; + } + LOG(INFO) << "Set download speed limit to " << max_speed; + download_speed_limit_ = max_speed; + td::actor::send_closure(download_speed_limiter_, &SpeedLimiter::set_max_speed, max_speed); + db_store_config(); +} + +void StorageManager::set_upload_speed_limit(double max_speed) { + if (max_speed < 0.0) { + max_speed = -1.0; + } + LOG(INFO) << "Set upload speed limit to " << max_speed; + upload_speed_limit_ = max_speed; + td::actor::send_closure(upload_speed_limiter_, &SpeedLimiter::set_max_speed, max_speed); + db_store_config(); +} + } // namespace ton \ No newline at end of file diff --git a/storage/storage-daemon/StorageManager.h b/storage/storage-daemon/StorageManager.h index 102e1b9a..fbfd68eb 100644 --- a/storage/storage-daemon/StorageManager.h +++ b/storage/storage-daemon/StorageManager.h @@ -22,6 +22,7 @@ #include "overlay/overlays.h" #include "storage/PeerManager.h" #include "storage/db.h" +#include "SpeedLimiter.h" namespace ton { @@ -63,6 +64,10 @@ class StorageManager : public td::actor::Actor { void wait_for_completion(td::Bits256 hash, td::Promise promise); void get_peers_info(td::Bits256 hash, td::Promise> promise); + void get_speed_limits(td::Promise> promise); // Download, upload + void set_download_speed_limit(double max_speed); + void set_upload_speed_limit(double max_speed); + private: adnl::AdnlNodeIdShort local_id_; std::string db_root_; @@ -89,6 +94,13 @@ class StorageManager : public td::actor::Actor { std::map torrents_; + double download_speed_limit_ = -1.0; + double upload_speed_limit_ = -1.0; + td::actor::ActorOwn download_speed_limiter_ = + td::actor::create_actor("DownloadRateLimitrer", -1.0); + td::actor::ActorOwn upload_speed_limiter_ = + td::actor::create_actor("DownloadRateLimitrer", -1.0); + td::Status add_torrent_impl(Torrent torrent, bool start_download, bool allow_upload); td::Result get_torrent(td::Bits256 hash) { @@ -102,9 +114,11 @@ class StorageManager : public td::actor::Actor { td::unique_ptr create_callback(td::Bits256 hash, std::shared_ptr closing_state); + void loaded_config_from_db(tl_object_ptr config); void load_torrents_from_db(std::vector torrents); void loaded_torrent_from_db(td::Bits256 hash, td::Result> R); void after_load_torrents_from_db(); + void db_store_config(); void db_store_torrent_list(); void on_torrent_closed(Torrent torrent, std::shared_ptr closing_state); diff --git a/storage/storage-daemon/StorageProvider.cpp b/storage/storage-daemon/StorageProvider.cpp index da4b5042..e1eb55d6 100644 --- a/storage/storage-daemon/StorageProvider.cpp +++ b/storage/storage-daemon/StorageProvider.cpp @@ -259,7 +259,7 @@ void StorageProvider::process_transaction(tl_object_ptr body = r_body.move_as_ok(); vm::CellSlice cs = vm::load_cell_slice(body); if (cs.size() >= 32) { - long long op_code = cs.prefetch_long(32); + long long op_code = cs.prefetch_ulong(32); // const op::offer_storage_contract = 0x107c49ef; -- old versions // const op::deploy_storage_contract = 0xe4748df1; -- new versions if((op_code == 0x107c49ef) || (op_code == 0xe4748df1)) { diff --git a/storage/storage-daemon/storage-daemon-cli.cpp b/storage/storage-daemon/storage-daemon-cli.cpp index 4d9345a8..9cb1d417 100644 --- a/storage/storage-daemon/storage-daemon-cli.cpp +++ b/storage/storage-daemon/storage-daemon-cli.cpp @@ -107,6 +107,29 @@ std::string size_to_str(td::uint64 size) { return s.as_cslice().str(); } +td::Result str_to_size(std::string s) { + if (!s.empty() && std::tolower(s.back()) == 'b') { + s.pop_back(); + } + int shift = 0; + if (!s.empty()) { + auto c = std::tolower(s.back()); + static std::string suffixes = "kmgtpe"; + for (size_t i = 0; i < suffixes.size(); ++i) { + if (c == suffixes[i]) { + shift = 10 * (int)(i + 1); + s.pop_back(); + break; + } + } + } + TRY_RESULT(x, td::to_integer_safe(s)); + if (td::count_leading_zeroes64(x) < shift) { + return td::Status::Error("Number is too big"); + } + return x << shift; +} + std::string time_to_str(td::uint32 time) { char time_buffer[80]; time_t rawtime = time; @@ -285,10 +308,13 @@ class StorageDaemonCli : public td::actor::Actor { } std::_Exit(0); } else if (tokens[0] == "help") { - if (tokens.size() != 1) { + std::string category; + if (tokens.size() == 2) { + category = tokens[1]; + } else if (tokens.size() != 1) { return td::Status::Error("Unexpected tokens"); } - return execute_help(); + return execute_help(category); } else if (tokens[0] == "setverbosity") { if (tokens.size() != 2) { return td::Status::Error("Expected level"); @@ -460,6 +486,46 @@ class StorageDaemonCli : public td::actor::Actor { return td::Status::Error("Unexpected EOLN"); } return execute_get_peers(hash, json); + } else if (tokens[0] == "get-pieces-info") { + td::Bits256 hash; + bool found_hash = false; + bool json = false; + bool files = false; + td::uint64 offset = 0; + td::optional max_pieces; + for (size_t i = 1; i < tokens.size(); ++i) { + if (!tokens[i].empty() && tokens[i][0] == '-') { + if (tokens[i] == "--json") { + json = true; + continue; + } + if (tokens[i] == "--files") { + files = true; + continue; + } + if (tokens[i] == "--offset") { + TRY_RESULT_PREFIX_ASSIGN(offset, td::to_integer_safe(tokens[i + 1]), "Invalid offset: "); + ++i; + continue; + } + if (tokens[i] == "--max-pieces") { + TRY_RESULT_PREFIX_ASSIGN(max_pieces, td::to_integer_safe(tokens[i + 1]), "Invalid offset: "); + max_pieces.value() = std::min(max_pieces.value(), ((td::uint64)1 << 62)); + ++i; + continue; + } + return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]); + } + if (found_hash) { + return td::Status::Error("Unexpected token"); + } + TRY_RESULT_ASSIGN(hash, parse_torrent(tokens[i])); + found_hash = true; + } + if (!found_hash) { + return td::Status::Error("Unexpected EOLN"); + } + return execute_get_pieces_info(hash, files, offset, max_pieces, json); } else if (tokens[0] == "download-pause" || tokens[0] == "download-resume") { if (tokens.size() != 2) { return td::Status::Error("Expected bag"); @@ -544,6 +610,48 @@ class StorageDaemonCli : public td::actor::Actor { return td::Status::Error("Unexpected EOLN"); } return execute_load_from(hash, std::move(meta), std::move(path)); + } else if (tokens[0] == "get-speed-limits") { + bool json = false; + for (size_t i = 1; i < tokens.size(); ++i) { + if (!tokens[i].empty() && tokens[i][0] == '-') { + if (tokens[i] == "--json") { + json = true; + continue; + } + return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]); + } + return td::Status::Error("Unexpected token"); + } + return execute_get_speed_limits(json); + } else if (tokens[0] == "set-speed-limits") { + td::optional download, upload; + for (size_t i = 1; i < tokens.size(); ++i) { + if (!tokens[i].empty() && tokens[i][0] == '-') { + if (tokens[i] == "--download") { + ++i; + if (tokens[i] == "unlimited") { + download = -1.0; + } else { + TRY_RESULT_PREFIX(x, str_to_size(tokens[i]), "Invalid download speed: "); + download = (double)x; + } + continue; + } + if (tokens[i] == "--upload") { + ++i; + if (tokens[i] == "unlimited") { + upload = -1.0; + } else { + TRY_RESULT_PREFIX(x, str_to_size(tokens[i]), "Invalid upload speed: "); + upload = (double)x; + } + continue; + } + return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]); + } + return td::Status::Error("Unexpected token"); + } + return execute_set_speed_limits(download, upload); } else if (tokens[0] == "new-contract-message") { td::Bits256 hash; std::string file; @@ -659,12 +767,12 @@ class StorageDaemonCli : public td::actor::Actor { continue; } if (tokens[i] == "--min-file-size") { - TRY_RESULT_PREFIX(x, td::to_integer_safe(tokens[i + 1]), "Invalid value for --min-file-size: "); + TRY_RESULT_PREFIX(x, str_to_size(tokens[i + 1]), "Invalid value for --min-file-size: "); new_params.minimal_file_size = x; continue; } if (tokens[i] == "--max-file-size") { - TRY_RESULT_PREFIX(x, td::to_integer_safe(tokens[i + 1]), "Invalid value for --max-file-size: "); + TRY_RESULT_PREFIX(x, str_to_size(tokens[i + 1]), "Invalid value for --max-file-size: "); new_params.maximal_file_size = x; continue; } @@ -708,7 +816,7 @@ class StorageDaemonCli : public td::actor::Actor { continue; } if (tokens[i] == "--max-total-size") { - TRY_RESULT_PREFIX(x, td::to_integer_safe(tokens[i + 1]), "Invalid value for --max-total-size: "); + TRY_RESULT_PREFIX(x, str_to_size(tokens[i + 1]), "Invalid value for --max-total-size: "); new_config.max_total_size = x; continue; } @@ -765,93 +873,111 @@ class StorageDaemonCli : public td::actor::Actor { } } - td::Status execute_help() { - td::TerminalIO::out() << "help\tPrint this help\n"; - td::TerminalIO::out() - << "create [-d description] [--no-upload] [--copy] [--json] \tCreate bag of files from \n"; - td::TerminalIO::out() << "\t-d\tDescription will be stored in torrent info\n"; - td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n"; - td::TerminalIO::out() << "\t--copy\tFiles will be copied to an internal directory of storage-daemon\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "add-by-hash [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 " - "file2 ...]\tAdd bag with given BagID (in hex)\n"; - td::TerminalIO::out() << "\t-d\tTarget directory, default is an internal directory of storage-daemon\n"; - td::TerminalIO::out() << "\t--paused\tDon't start download immediately\n"; - td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n"; - td::TerminalIO::out() - << "\t--partial\tEverything after this flag is a list of filenames. Only these files will be downloaded.\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "add-by-meta [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 " - "file2 ...]\tLoad meta from file and add bag\n"; - td::TerminalIO::out() << "\tFlags are the same as in add-by-hash\n"; - td::TerminalIO::out() << "list [--hashes] [--json]\tPrint list of bags\n"; - td::TerminalIO::out() << "\t--hashes\tPrint full BagID\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "get [--json]\tPrint information about \n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "\tHere and below bags are identified by BagID (in hex) or index (see bag list)\n"; - td::TerminalIO::out() << "get-meta \tSave bag meta of to \n"; - td::TerminalIO::out() << "get-peers [--json]\tPrint a list of peers\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "download-pause \tPause download of \n"; - td::TerminalIO::out() << "download-resume \tResume download of \n"; - td::TerminalIO::out() << "upload-pause \tPause upload of \n"; - td::TerminalIO::out() << "upload-resume \tResume upload of \n"; - td::TerminalIO::out() << "priority-all

\tSet priority of all files in to

\n"; - td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; - td::TerminalIO::out() << "priority-idx

\tSet priority of file # in to

\n"; - td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; - td::TerminalIO::out() << "priority-name

\tSet priority of file in to

\n"; - td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; - td::TerminalIO::out() << "remove [--remove-files]\tRemove \n"; - td::TerminalIO::out() << "\t--remove-files - also remove all files\n"; - td::TerminalIO::out() << "load-from [--meta meta] [--files path]\tProvide meta and data for an existing " - "incomplete bag.\n"; - td::TerminalIO::out() << "\t--meta meta\ttorrent info and header will be inited (if not ready) from meta file\n"; - td::TerminalIO::out() << "\t--files path\tdata for files will be taken from here\n"; - td::TerminalIO::out() << "new-contract-message [--query-id id] --provider \tCreate " - "\"new contract message\" for storage provider. Saves message body to .\n"; - td::TerminalIO::out() << "\t\tAddress of storage provider account to take parameters from.\n"; - td::TerminalIO::out() << "new-contract-message [--query-id id] --rate --max-span " - "\tSame thing, but parameters are not fetched automatically.\n"; - td::TerminalIO::out() << "exit\tExit\n"; - td::TerminalIO::out() << "quit\tExit\n"; - td::TerminalIO::out() << "setverbosity \tSet vetbosity to in [0..10]\n"; - td::TerminalIO::out() << "\nStorage provider control:\n"; - td::TerminalIO::out() << "import-pk \tImport private key from \n"; - td::TerminalIO::out() << "deploy-provider\tInit storage provider by deploying a new provider smart contract\n"; - td::TerminalIO::out() - << "init-provider \tInit storage provider using the existing provider smart contract\n"; - td::TerminalIO::out() << "remove-storage-provider\tRemove storage provider\n"; - td::TerminalIO::out() - << "\tSmart contracts in blockchain and bags will remain intact, but they will not be managed anymore\n"; - td::TerminalIO::out() << "get-provider-params [address] [--json]\tPrint parameters of the smart contract\n"; - td::TerminalIO::out() - << "\taddress\tAddress of a smart contract. Default is the provider managed by this daemon.\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "set-provider-params [--accept x] [--rate x] [--max-span x] [--min-file-size x] " - "[--max-file-size x]\tSet parameters of the smart contract\n"; - td::TerminalIO::out() << "\t--accept\tAccept new contracts: 0 (no) or 1 (yes)\n"; - td::TerminalIO::out() << "\t--rate\tPrice of storage, nanoTON per MB*day\n"; - td::TerminalIO::out() << "\t--max-span\n"; - td::TerminalIO::out() << "\t--min-file-size\tMinimal total size of a bag of files (bytes)\n"; - td::TerminalIO::out() << "\t--max-file-size\tMaximal total size of a bag of files (bytes)\n"; - td::TerminalIO::out() - << "get-provider-info [--balances] [--contracts] [--json]\tPrint information about storage provider\n"; - td::TerminalIO::out() << "\t--contracts\tPrint list of storage contracts\n"; - td::TerminalIO::out() << "\t--balances\tPrint balances of the main contract and storage contracts\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() - << "set-provider-config [--max-contracts x] [--max-total-size x]\tSet configuration parameters\n"; - td::TerminalIO::out() << "\t--max-contracts\tMaximal number of storage contracts\n"; - td::TerminalIO::out() << "\t--max-total-size\tMaximal total size storage contracts (in bytes)\n"; - td::TerminalIO::out() << "withdraw

\tSend bounty from storage contract
to the main contract\n"; - td::TerminalIO::out() << "withdraw-all\tSend bounty from all storage contracts (where at least 1 TON is available) " - "to the main contract\n"; - td::TerminalIO::out() - << "send-coins
[--message msg]\tSend nanoTON to
from the main contract\n"; - td::TerminalIO::out() - << "close-contract
\tClose storage contract
and delete bag (if possible)\n"; + td::Status execute_help(std::string category) { + if (category == "") { + td::TerminalIO::out() << "create [-d description] [--no-upload] [--copy] [--json] \tCreate bag of " + "files from \n"; + td::TerminalIO::out() << "\t-d\tDescription will be stored in torrent info\n"; + td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n"; + td::TerminalIO::out() << "\t--copy\tFiles will be copied to an internal directory of storage-daemon\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "add-by-hash [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 " + "file2 ...]\tAdd bag with given BagID (in hex)\n"; + td::TerminalIO::out() << "\t-d\tTarget directory, default is an internal directory of storage-daemon\n"; + td::TerminalIO::out() << "\t--paused\tDon't start download immediately\n"; + td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n"; + td::TerminalIO::out() + << "\t--partial\tEverything after this flag is a list of filenames. Only these files will be downloaded.\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "add-by-meta [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 " + "file2 ...]\tLoad meta from file and add bag\n"; + td::TerminalIO::out() << "\tFlags are the same as in add-by-hash\n"; + td::TerminalIO::out() << "list [--hashes] [--json]\tPrint list of bags\n"; + td::TerminalIO::out() << "\t--hashes\tPrint full BagID\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "get [--json]\tPrint information about \n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "\tHere and below bags are identified by BagID (in hex) or index (see bag list)\n"; + td::TerminalIO::out() << "get-meta \tSave bag meta of to \n"; + td::TerminalIO::out() << "get-peers [--json]\tPrint a list of peers\n"; + td::TerminalIO::out() << "get-pieces-info [--files] [--offset l] [--max-pieces m] [--json]\tPrint " + "information about ready pieces\n"; + td::TerminalIO::out() << "\t--files\tShow piece ranges for each file\n"; + td::TerminalIO::out() << "\t--offset l\tShow pieces starting from l (deafault: 0)\n"; + td::TerminalIO::out() << "\t--max-pieces m\tShow no more than m pieces (deafault: unlimited)\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "download-pause \tPause download of \n"; + td::TerminalIO::out() << "download-resume \tResume download of \n"; + td::TerminalIO::out() << "upload-pause \tPause upload of \n"; + td::TerminalIO::out() << "upload-resume \tResume upload of \n"; + td::TerminalIO::out() << "priority-all

\tSet priority of all files in to

\n"; + td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; + td::TerminalIO::out() << "priority-idx

\tSet priority of file # in to

\n"; + td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; + td::TerminalIO::out() << "priority-name

\tSet priority of file in to

\n"; + td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; + td::TerminalIO::out() << "remove [--remove-files]\tRemove \n"; + td::TerminalIO::out() << "\t--remove-files - also remove all files\n"; + td::TerminalIO::out() << "load-from [--meta meta] [--files path]\tProvide meta and data for an existing " + "incomplete bag.\n"; + td::TerminalIO::out() << "\t--meta meta\ttorrent info and header will be inited (if not ready) from meta file\n"; + td::TerminalIO::out() << "\t--files path\tdata for files will be taken from here\n"; + td::TerminalIO::out() << "get-speed-limits [--json]\tShow global limits for download and upload speed\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() + << "set-speed-limits [--download x] [--upload x]\tSet global limits for download and upload speed\n"; + td::TerminalIO::out() << "\t--download x\tDownload speed limit in bytes/s, or \"unlimited\"\n"; + td::TerminalIO::out() << "\t--upload x\tUpload speed limit in bytes/s, or \"unlimited\"\n"; + td::TerminalIO::out() << "new-contract-message [--query-id id] --provider \tCreate " + "\"new contract message\" for storage provider. Saves message body to .\n"; + td::TerminalIO::out() << "\t\tAddress of storage provider account to take parameters from.\n"; + td::TerminalIO::out() << "new-contract-message [--query-id id] --rate --max-span " + "\tSame thing, but parameters are not fetched automatically.\n"; + td::TerminalIO::out() << "exit\tExit\n"; + td::TerminalIO::out() << "quit\tExit\n"; + td::TerminalIO::out() << "setverbosity \tSet vetbosity to in [0..10]\n"; + td::TerminalIO::out() << "help\tPrint this help\n"; + td::TerminalIO::out() << "help provider\tcommands for deploying and controling storage provider\n"; + } else if (category == "provider") { + td::TerminalIO::out() << "\nStorage provider control:\n"; + td::TerminalIO::out() << "import-pk \tImport private key from \n"; + td::TerminalIO::out() << "deploy-provider\tInit storage provider by deploying a new provider smart contract\n"; + td::TerminalIO::out() + << "init-provider \tInit storage provider using the existing provider smart contract\n"; + td::TerminalIO::out() << "remove-storage-provider\tRemove storage provider\n"; + td::TerminalIO::out() + << "\tSmart contracts in blockchain and bags will remain intact, but they will not be managed anymore\n"; + td::TerminalIO::out() << "get-provider-params [address] [--json]\tPrint parameters of the smart contract\n"; + td::TerminalIO::out() + << "\taddress\tAddress of a smart contract. Default is the provider managed by this daemon.\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "set-provider-params [--accept x] [--rate x] [--max-span x] [--min-file-size x] " + "[--max-file-size x]\tSet parameters of the smart contract\n"; + td::TerminalIO::out() << "\t--accept\tAccept new contracts: 0 (no) or 1 (yes)\n"; + td::TerminalIO::out() << "\t--rate\tPrice of storage, nanoTON per MB*day\n"; + td::TerminalIO::out() << "\t--max-span\n"; + td::TerminalIO::out() << "\t--min-file-size\tMinimal total size of a bag of files (bytes)\n"; + td::TerminalIO::out() << "\t--max-file-size\tMaximal total size of a bag of files (bytes)\n"; + td::TerminalIO::out() + << "get-provider-info [--balances] [--contracts] [--json]\tPrint information about storage provider\n"; + td::TerminalIO::out() << "\t--contracts\tPrint list of storage contracts\n"; + td::TerminalIO::out() << "\t--balances\tPrint balances of the main contract and storage contracts\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() + << "set-provider-config [--max-contracts x] [--max-total-size x]\tSet configuration parameters\n"; + td::TerminalIO::out() << "\t--max-contracts\tMaximal number of storage contracts\n"; + td::TerminalIO::out() << "\t--max-total-size\tMaximal total size storage contracts (in bytes)\n"; + td::TerminalIO::out() << "withdraw

\tSend bounty from storage contract
to the main contract\n"; + td::TerminalIO::out() + << "withdraw-all\tSend bounty from all storage contracts (where at least 1 TON is available) " + "to the main contract\n"; + td::TerminalIO::out() << "send-coins
[--message msg]\tSend nanoTON to
from " + "the main contract\n"; + td::TerminalIO::out() + << "close-contract
\tClose storage contract
and delete bag (if possible)\n"; + } else { + td::TerminalIO::out() << "Unknown command 'help " << category << "'\n"; + } command_finished(td::Status::OK()); return td::Status::OK(); } @@ -871,7 +997,7 @@ class StorageDaemonCli : public td::actor::Actor { td::Status execute_create(std::string path, std::string description, bool upload, bool copy, bool json) { TRY_RESULT_PREFIX_ASSIGN(path, td::realpath(path), "Invalid path: "); - auto query = create_tl_object(path, description, upload, copy); + auto query = create_tl_object(path, description, upload, copy, 0); send_query(std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -904,7 +1030,7 @@ class StorageDaemonCli : public td::actor::Actor { } } auto query = create_tl_object(hash, std::move(root_dir), !paused, upload, - std::move(priorities)); + std::move(priorities), 0); send_query(std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -938,7 +1064,7 @@ class StorageDaemonCli : public td::actor::Actor { } } auto query = create_tl_object(std::move(meta), std::move(root_dir), !paused, - upload, std::move(priorities)); + upload, std::move(priorities), 0); send_query(std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -957,7 +1083,7 @@ class StorageDaemonCli : public td::actor::Actor { } td::Status execute_list(bool with_hashes, bool json) { - auto query = create_tl_object(); + auto query = create_tl_object(0); send_query(std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -975,7 +1101,7 @@ class StorageDaemonCli : public td::actor::Actor { } td::Status execute_get(td::Bits256 hash, bool json) { - auto query = create_tl_object(hash); + auto query = create_tl_object(hash, 0); send_query(std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -993,7 +1119,7 @@ class StorageDaemonCli : public td::actor::Actor { } td::Status execute_get_meta(td::Bits256 hash, std::string meta_file) { - auto query = create_tl_object(hash); + auto query = create_tl_object(hash, 0); send_query(std::move(query), [SelfId = actor_id(this), meta_file](td::Result> R) { if (R.is_error()) { @@ -1014,7 +1140,7 @@ class StorageDaemonCli : public td::actor::Actor { } td::Status execute_get_peers(td::Bits256 hash, bool json) { - auto query = create_tl_object(hash); + auto query = create_tl_object(hash, 0); send_query( std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -1054,6 +1180,63 @@ class StorageDaemonCli : public td::actor::Actor { return td::Status::OK(); } + td::Status execute_get_pieces_info(td::Bits256 hash, bool files, td::uint64 offset, + td::optional max_pieces, bool json) { + auto query = create_tl_object(hash, files ? 1 : 0, offset, + max_pieces ? max_pieces.value() : -1); + send_query(std::move(query), + [=, SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + return; + } + if (json) { + print_json(R.ok()); + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + return; + } + auto obj = R.move_as_ok(); + td::TerminalIO::out() << "BagID " << hash.to_hex() << "\n"; + td::TerminalIO::out() << "Total pieces: " << obj->total_pieces_ << ", piece size: " << obj->piece_size_ + << "\n"; + if (files) { + if (obj->flags_ & 1) { + td::TerminalIO::out() << "Files:\n"; + std::vector> table; + table.push_back({"#####", "Piece range", "Name"}); + size_t i = 0; + for (const auto& f : obj->files_) { + table.push_back({i == 0 ? "" : td::to_string(i - 1), + PSTRING() << "[" << f->range_l_ << ".." << f->range_r_ << ")", + f->name_.empty() ? "[HEADER]" : f->name_}); + ++i; + } + print_table(table, {1, 2}); + } else { + td::TerminalIO::out() << "Cannot show files: torrent header is not available\n"; + } + } + td::uint64 l = obj->range_l_, r = obj->range_r_; + td::TerminalIO::out() << "Pieces [" << l << ".." << r << ")\n"; + if (obj->range_l_ != obj->range_r_) { + std::vector> table; + td::uint64 i = l; + while (i < r) { + td::uint64 ir = std::min(i + 100, r); + std::string s = "["; + for (td::uint64 j = i; j < ir; ++j) { + s += (obj->piece_ready_bitset_[(j - l) / 8] & (1 << ((j - l) % 8)) ? '#' : '-'); + } + s += ']'; + table.push_back({std::to_string(i), s}); + i = ir; + } + print_table(table, {1}); + } + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + }); + return td::Status::OK(); + } + td::Status execute_set_active_download(td::Bits256 hash, bool active) { auto query = create_tl_object(hash, active); send_query(std::move(query), @@ -1156,7 +1339,7 @@ class StorageDaemonCli : public td::actor::Actor { if (!path.empty()) { TRY_RESULT_PREFIX_ASSIGN(path, td::realpath(path), "Invalid path: "); } - auto query = create_tl_object(hash, std::move(meta_data), std::move(path)); + auto query = create_tl_object(hash, std::move(meta_data), std::move(path), 0); send_query(std::move(query), [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -1183,6 +1366,59 @@ class StorageDaemonCli : public td::actor::Actor { return td::Status::OK(); } + td::Status execute_get_speed_limits(bool json) { + auto query = create_tl_object(0); + send_query(std::move(query), [=, SelfId = actor_id(this)]( + td::Result> R) { + if (R.is_error()) { + return; + } + if (json) { + print_json(R.ok()); + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + return; + } + auto obj = R.move_as_ok(); + if (obj->download_ < 0.0) { + td::TerminalIO::out() << "Download speed limit: unlimited\n"; + } else { + td::TerminalIO::out() << "Download speed limit: " << td::format::as_size((td::uint64)obj->download_) << "/s\n"; + } + if (obj->upload_ < 0.0) { + td::TerminalIO::out() << "Upload speed limit: unlimited\n"; + } else { + td::TerminalIO::out() << "Upload speed limit: " << td::format::as_size((td::uint64)obj->upload_) << "/s\n"; + } + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + }); + return td::Status::OK(); + } + + td::Status execute_set_speed_limits(td::optional download, td::optional upload) { + if (!download && !upload) { + return td::Status::Error("No parameters are set"); + } + auto query = create_tl_object(); + query->flags_ = 0; + if (download) { + query->flags_ |= 1; + query->download_ = download.value(); + } + if (upload) { + query->flags_ |= 2; + query->upload_ = upload.value(); + } + send_query(std::move(query), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + return; + } + td::TerminalIO::out() << "Speed limits were set\n"; + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + }); + return td::Status::OK(); + } + td::Status execute_new_contract_message(td::Bits256 hash, std::string file, td::uint64 query_id, td::optional provider_address, td::optional rate, td::optional max_span) { @@ -1282,25 +1518,26 @@ class StorageDaemonCli : public td::actor::Actor { td::Status execute_get_provider_params(std::string address, bool json) { auto query = create_tl_object(address); - send_query(std::move(query), - [=, SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - return; - } - if (json) { - print_json(R.ok()); - td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); - return; - } - auto params = R.move_as_ok(); - td::TerminalIO::out() << "Storage provider parameters:\n"; - td::TerminalIO::out() << "Accept new contracts: " << params->accept_new_contracts_ << "\n"; - td::TerminalIO::out() << "Rate (nanoTON per day*MB): " << params->rate_per_mb_day_ << "\n"; - td::TerminalIO::out() << "Max span: " << (td::uint32)params->max_span_ << "\n"; - td::TerminalIO::out() << "Min file size: " << (td::uint64)params->minimal_file_size_ << "\n"; - td::TerminalIO::out() << "Max file size: " << (td::uint64)params->maximal_file_size_ << "\n"; - td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); - }); + send_query(std::move(query), [=, SelfId = actor_id(this)]( + td::Result> R) { + if (R.is_error()) { + return; + } + if (json) { + print_json(R.ok()); + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + return; + } + auto params = R.move_as_ok(); + td::TerminalIO::out() << "Storage provider parameters:\n"; + td::TerminalIO::out() << "Accept new contracts: " << params->accept_new_contracts_ << "\n"; + td::TerminalIO::out() << "Rate (nanoTON per day*MB): " << params->rate_per_mb_day_ << "\n"; + td::TerminalIO::out() << "Max span: " << (td::uint32)params->max_span_ << "\n"; + auto min_size = (td::uint64)params->minimal_file_size_, max_size = (td::uint64)params->maximal_file_size_; + td::TerminalIO::out() << "Min file size: " << td::format::as_size(min_size) << " (" << min_size << ")\n"; + td::TerminalIO::out() << "Max file size: " << td::format::as_size(max_size) << " (" << max_size << ")\n"; + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + }); return td::Status::OK(); } @@ -1632,6 +1869,7 @@ class StorageDaemonCli : public td::actor::Actor { add_id(obj.torrent_->hash_); td::TerminalIO::out() << "BagID = " << obj.torrent_->hash_.to_hex() << "\n"; td::TerminalIO::out() << "Index = " << hash_to_id_[obj.torrent_->hash_] << "\n"; + td::TerminalIO::out() << "Added: " << time_to_str(obj.torrent_->added_at_) << "\n"; if (obj.torrent_->flags_ & 4) { // fatal error td::TerminalIO::out() << "FATAL ERROR: " << obj.torrent_->fatal_error_ << "\n"; } diff --git a/storage/storage-daemon/storage-daemon.cpp b/storage/storage-daemon/storage-daemon.cpp index 7563332a..372829f4 100644 --- a/storage/storage-daemon/storage-daemon.cpp +++ b/storage/storage-daemon/storage-daemon.cpp @@ -180,6 +180,9 @@ class StorageDaemon : public td::actor::Actor { dht_id_ = dht_id_full.compute_short_id(); td::actor::send_closure(adnl_, &adnl::Adnl::add_id, dht_id_full, addr_list, static_cast(0)); + LOG(INFO) << "Storage daemon ADNL id is " << local_id_; + LOG(INFO) << "DHT id is " << dht_id_; + if (client_mode_) { auto D = dht::Dht::create_client(dht_id_, db_root_, dht_config_, keyring_.get(), adnl_.get()); D.ensure(); @@ -431,6 +434,52 @@ class StorageDaemon : public td::actor::Actor { })); } + void run_control_query(ton_api::storage_daemon_getTorrentPiecesInfo &query, td::Promise promise) { + td::Bits256 hash = query.hash_; + td::actor::send_closure( + manager_, &StorageManager::with_torrent, hash, + promise.wrap([query = std::move(query)](NodeActor::NodeState state) -> td::Result { + Torrent &torrent = state.torrent; + if (!torrent.inited_info()) { + return td::Status::Error("Torrent info is not available"); + } + td::uint64 total_pieces = torrent.get_info().pieces_count(); + td::uint64 range_l = std::min(total_pieces, query.offset_); + td::uint64 size = query.max_pieces_ != -1 ? std::min(total_pieces - range_l, query.max_pieces_) + : total_pieces - range_l; + td::BufferSlice piece_ready((size + 7) / 8); + std::fill(piece_ready.data(), piece_ready.data() + piece_ready.size(), 0); + for (td::uint64 i = range_l; i < range_l + size; ++i) { + if (torrent.is_piece_ready(i)) { + piece_ready.data()[(i - range_l) / 8] |= (1 << ((i - range_l) % 8)); + } + } + + auto result = create_tl_object(); + result->total_pieces_ = total_pieces; + result->piece_size_ = torrent.get_info().piece_size; + result->range_l_ = range_l; + result->range_r_ = range_l + size; + result->piece_ready_bitset_ = std::move(piece_ready); + + if ((query.flags_ & 1) && torrent.inited_header()) { + result->flags_ = 1; + auto range = torrent.get_header_parts_range(); + result->files_.push_back( + create_tl_object("", range.begin, range.end)); + for (size_t i = 0; i < torrent.get_files_count().value(); ++i) { + auto range = torrent.get_file_parts_range(i); + result->files_.push_back(create_tl_object( + torrent.get_file_name(i).str(), range.begin, range.end)); + } + } else { + result->flags_ = 0; + } + + return serialize_tl_object(result, true); + })); + } + void run_control_query(ton_api::storage_daemon_setFilePriorityAll &query, td::Promise promise) { TRY_RESULT_PROMISE(promise, priority, td::narrow_cast_safe(query.priority_)); td::actor::send_closure(manager_, &StorageManager::set_all_files_priority, query.hash_, priority, @@ -491,6 +540,24 @@ class StorageDaemon : public td::actor::Actor { }); } + void run_control_query(ton_api::storage_daemon_getSpeedLimits &query, td::Promise promise) { + td::actor::send_closure(manager_, &StorageManager::get_speed_limits, + promise.wrap([](std::pair limits) -> td::BufferSlice { + return create_serialize_tl_object(limits.first, + limits.second); + })); + } + + void run_control_query(ton_api::storage_daemon_setSpeedLimits &query, td::Promise promise) { + if (query.flags_ & 1) { + td::actor::send_closure(manager_, &StorageManager::set_download_speed_limit, query.download_); + } + if (query.flags_ & 2) { + td::actor::send_closure(manager_, &StorageManager::set_upload_speed_limit, query.upload_); + } + promise.set_result(create_serialize_tl_object()); + } + void run_control_query(ton_api::storage_daemon_getNewContractMessage &query, td::Promise promise) { td::Promise> P = [promise = std::move(promise), hash = query.hash_, query_id = query.query_id_, @@ -779,6 +846,7 @@ class StorageDaemon : public td::actor::Actor { file->name_ = torrent.get_file_name(i).str(); file->size_ = torrent.get_file_size(i); file->downloaded_size_ = torrent.get_file_ready_size(i); + file->flags_ = 0; obj.files_.push_back(std::move(file)); } } @@ -798,6 +866,7 @@ class StorageDaemon : public td::actor::Actor { obj->active_upload_ = state.active_upload; obj->download_speed_ = state.download_speed; obj->upload_speed_ = state.upload_speed; + obj->added_at_ = state.added_at; promise.set_result(std::move(obj)); }); } @@ -816,6 +885,7 @@ class StorageDaemon : public td::actor::Actor { obj->torrent_->active_upload_ = state.active_upload; obj->torrent_->download_speed_ = state.download_speed; obj->torrent_->upload_speed_ = state.upload_speed; + obj->torrent_->added_at_ = state.added_at; for (size_t i = 0; i < obj->files_.size(); ++i) { obj->files_[i]->priority_ = (i < state.file_priority.size() ? state.file_priority[i] : 1); diff --git a/storage/test/storage.cpp b/storage/test/storage.cpp index b4f67b9b..e7a97352 100644 --- a/storage/test/storage.cpp +++ b/storage/test/storage.cpp @@ -1342,7 +1342,7 @@ TEST(Torrent, Peer) { guard->push_back(td::actor::create_actor( "Node#1", 1, std::move(torrent), td::make_unique(stop_watcher, complete_watcher), - td::make_unique(peer_manager.get(), 1, gen_peers(1, 2)), nullptr)); + td::make_unique(peer_manager.get(), 1, gen_peers(1, 2)), nullptr, ton::SpeedLimiters{})); for (size_t i = 2; i <= peers_n; i++) { ton::Torrent::Options options; options.in_memory = true; @@ -1351,7 +1351,7 @@ TEST(Torrent, Peer) { PSLICE() << "Node#" << i, i, std::move(other_torrent), td::make_unique(stop_watcher, complete_watcher), td::make_unique(peer_manager.get(), i, gen_peers(i, 2)), - nullptr); + nullptr, ton::SpeedLimiters{}); if (i == 3) { td::actor::create_actor("StatsActor", node_actor.get()).release(); diff --git a/tdutils/td/utils/OptionParser.cpp b/tdutils/td/utils/OptionParser.cpp index b9584856..634570e1 100644 --- a/tdutils/td/utils/OptionParser.cpp +++ b/tdutils/td/utils/OptionParser.cpp @@ -33,7 +33,7 @@ void OptionParser::set_description(string description) { void OptionParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description, std::function callback) { for (auto &option : options_) { - if (option.short_key == short_key || (!long_key.empty() && long_key == option.long_key)) { + if ((short_key != '\0' && option.short_key == short_key) || (!long_key.empty() && long_key == option.long_key)) { LOG(ERROR) << "Ignore duplicated option '" << short_key << "' '" << long_key << "'"; } } diff --git a/tdutils/td/utils/port/config.h b/tdutils/td/utils/port/config.h index 77143668..f89082f3 100644 --- a/tdutils/td/utils/port/config.h +++ b/tdutils/td/utils/port/config.h @@ -28,35 +28,33 @@ #define TD_PORT_POSIX 1 #endif -#if TD_LINUX || TD_ANDROID || TD_TIZEN +#if TD_EMSCRIPTEN + #define TD_POLL_POLL 1 +#elif TD_LINUX || TD_ANDROID || TD_TIZEN #define TD_POLL_EPOLL 1 - #define TD_EVENTFD_LINUX 1 #elif TD_FREEBSD || TD_OPENBSD || TD_NETBSD #define TD_POLL_KQUEUE 1 - #define TD_EVENTFD_BSD 1 #elif TD_CYGWIN #define TD_POLL_SELECT 1 - #define TD_EVENTFD_BSD 1 -#elif TD_EMSCRIPTEN - #define TD_POLL_POLL 1 - // #define TD_EVENTFD_UNSUPPORTED 1 #elif TD_DARWIN #define TD_POLL_KQUEUE 1 - #define TD_EVENTFD_BSD 1 #elif TD_WINDOWS #define TD_POLL_WINEVENT 1 - #define TD_EVENTFD_WINDOWS 1 #else #error "Poll's implementation is not defined" #endif -#if TD_EMSCRIPTEN - // #define TD_THREAD_UNSUPPORTED 1 - #define TD_POLL_EPOLL 1 - #define TD_EVENTFD_UNSUPPORTED 0 - #define TD_THREAD_PTHREAD 1 +#if TD_LINUX || TD_ANDROID || TD_TIZEN #define TD_EVENTFD_LINUX 1 -#elif TD_TIZEN || TD_LINUX || TD_DARWIN +#elif TD_FREEBSD || TD_OPENBSD || TD_NETBSD || TD_CYGWIN || TD_DARWIN + #define TD_EVENTFD_BSD 1 +#elif TD_WINDOWS + #define TD_EVENTFD_WINDOWS 1 +#else + #error "eventfd's implementation is not defined" +#endif + +#if TD_TIZEN || TD_LINUX || TD_DARWIN || TD_EMSCRIPTEN #define TD_THREAD_PTHREAD 1 #else #define TD_THREAD_STL 1 diff --git a/tdutils/td/utils/port/platform.h b/tdutils/td/utils/port/platform.h index 783dd399..a7b19220 100644 --- a/tdutils/td/utils/port/platform.h +++ b/tdutils/td/utils/port/platform.h @@ -20,6 +20,11 @@ // clang-format off +/*** Determine emscripten ***/ +#if defined(__EMSCRIPTEN__) + #define TD_EMSCRIPTEN 1 +#endif + /*** Platform macros ***/ #if defined(_WIN32) || defined(_WINDOWS) // _WINDOWS is defined by CMake #if defined(__cplusplus_winrt) @@ -63,10 +68,11 @@ #define TD_NETBSD 1 #elif defined(__CYGWIN__) #define TD_CYGWIN 1 -#elif defined(__EMSCRIPTEN__) - #define TD_EMSCRIPTEN 1 -#elif defined(__unix__) // all unices not caught above - #warning "Probably unsupported Unix platform. Feel free to try to compile" +#elif defined(__unix__) // all unices not caught above + // supress if emscripten + #if !TD_EMSCRIPTEN + #warning "Probably unsupported Unix platform. Feel free to try to compile" + #endif #define TD_CYGWIN 1 #else #error "Probably unsupported platform. Feel free to remove the error and try to recompile" diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 93fa16ae..61191ea4 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -6,9 +6,14 @@ Test_Fift_bug_div_default 1ac42861ce96b2896001c587f65e9afe1617db48859f19c2f4e306 Test_Fift_bug_newlize_default e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 Test_Fift_bug_ufits_default 51bf5a9f1ed7633a193f6fdd17a7a3af8e032dfe72a9669c85e8639aa8a7c195 Test_Fift_contfrac_default 09ebce5c91bcb70696c6fb6981d82dc3b9e3444dab608a7a1b044c0ddd778a96 +Test_Fift_test_asm_nested_program_default 2a19decac67adb719c444ab42879a5d894447d450d1998848c469605531076ad Test_Fift_test_default 4e44b3382963ec89f7b5c8f2ebd85da3bc8aebad5b49f5b11b14075061477b4d Test_Fift_test_dict_default a9c8cbcfdece5573185022cea07f59f1bc404e5d879e5157a5745757f8ee0525 +Test_Fift_test_disasm_default dacaa555f5f217b2373e01e3bcd59634e480f5759dcc43edbdef35273ae38492 +Test_Fift_test_fiftext_default 2b0db5d4d4bfbc705b959cc787540d7b3a21a71469eac54756e76953f0d9afca Test_Fift_test_fixed_default 278a19d56b773102caf5c1fe2997ea6c8d0d9e720eff8503feede6398a197eec +Test_Fift_test_hmap_default c269246882039824bb5822e896c3e6e82ef8e1251b6b251f5af8ea9fb8d05067 +Test_Fift_test_namespaces_default e6419619c51332fb5e8bf22043ef415db686c47fe24f03061e5ad831014e7c6c Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a Test_Fift_test_sort_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a Test_Fift_testvm2_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index a01da11a..ce4d2731 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -44,6 +44,7 @@ liteServer.transactionList ids:(vector tonNode.blockIdExt) transactions:bytes = liteServer.transactionId mode:# account:mode.0?int256 lt:mode.1?long hash:mode.2?int256 = liteServer.TransactionId; liteServer.transactionId3 account:int256 lt:long = liteServer.TransactionId3; liteServer.blockTransactions id:tonNode.blockIdExt req_count:# incomplete:Bool ids:(vector liteServer.transactionId) proof:bytes = liteServer.BlockTransactions; +liteServer.blockTransactionsExt id:tonNode.blockIdExt req_count:# incomplete:Bool transactions:bytes proof:bytes = liteServer.BlockTransactionsExt; liteServer.signature node_id_short:int256 signature:bytes = liteServer.Signature; liteServer.signatureSet validator_set_hash:int catchain_seqno:int signatures:(vector liteServer.signature) = liteServer.SignatureSet; liteServer.blockLinkBack to_key_block:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt dest_proof:bytes proof:bytes state_proof:bytes = liteServer.BlockLink; @@ -76,6 +77,7 @@ liteServer.getOneTransaction id:tonNode.blockIdExt account:liteServer.accountId liteServer.getTransactions count:# account:liteServer.accountId lt:long hash:int256 = liteServer.TransactionList; liteServer.lookupBlock mode:# id:tonNode.blockId lt:mode.1?long utime:mode.2?int = liteServer.BlockHeader; liteServer.listBlockTransactions id:tonNode.blockIdExt mode:# count:# after:mode.7?liteServer.transactionId3 reverse_order:mode.6?true want_proof:mode.5?true = liteServer.BlockTransactions; +liteServer.listBlockTransactionsExt id:tonNode.blockIdExt mode:# count:# after:mode.7?liteServer.transactionId3 reverse_order:mode.6?true want_proof:mode.5?true = liteServer.BlockTransactionsExt; liteServer.getBlockProof mode:# known_block:tonNode.blockIdExt target_block:mode.0?tonNode.blockIdExt = liteServer.PartialBlockProof; liteServer.getConfigAll mode:# id:tonNode.blockIdExt = liteServer.ConfigInfo; liteServer.getConfigParams mode:# id:tonNode.blockIdExt param_list:(vector int) = liteServer.ConfigInfo; diff --git a/tl/generate/scheme/lite_api.tlo b/tl/generate/scheme/lite_api.tlo index 0572fd46..da64ac53 100644 Binary files a/tl/generate/scheme/lite_api.tlo and b/tl/generate/scheme/lite_api.tlo differ diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 643825ea..66e6f4a6 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -588,12 +588,14 @@ engine.gc ids:(vector int256) = engine.Gc; engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster; engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; +engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) + fullnodeconfig:engine.validator.fullNodeConfig liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) - gc:engine.gc = engine.validator.Config; + gc:engine.gc = engine.validator.Config; ---functions--- ---types--- @@ -642,6 +644,8 @@ engine.validator.onePerfTimerStat time:int min:double avg:double max:double = en engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByName) = engine.validator.PerfTimerStats; +engine.validator.shardOutQueueSize size:int = engine.validator.ShardOutQueueSize; + ---functions--- @@ -693,6 +697,8 @@ engine.validator.signShardOverlayCertificate workchain:int shard:long signed_key engine.validator.importShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; engine.validator.getPerfTimerStats name:string = engine.validator.PerfTimerStats; +engine.validator.getShardOutQueueSize flags:# block_id:tonNode.blockId dest_wc:flags.0?int dest_shard:flags.0?long = engine.validator.ShardOutQueueSize; +engine.validator.setExtMessagesBroadcastDisabled disabled:Bool = engine.validator.Success; ---types--- @@ -758,9 +764,12 @@ storage.db.key.torrentMeta hash:int256 = storage.db.key.TorrentMeta; storage.db.key.priorities hash:int256 = storage.db.key.Priorities; storage.db.key.piecesInDb hash:int256 = storage.db.key.PiecesInDb; storage.db.key.pieceInDb hash:int256 idx:long = storage.db.key.PieceInDb; +storage.db.key.config = storage.db.key.Config; +storage.db.config flags:# download_speed_limit:double upload_speed_limit:double = storage.db.Config; storage.db.torrentList torrents:(vector int256) = storage.db.TorrentList; storage.db.torrent root_dir:string active_download:Bool active_upload:Bool = storage.db.TorrentShort; +storage.db.torrentV2 flags:# root_dir:string added_at:int active_download:Bool active_upload:Bool = storage.db.TorrentShort; storage.db.priorities actions:(vector storage.PriorityAction) = storage.db.Priorities; storage.db.piecesInDb pieces:(vector long) = storage.db.PiecesInDb; @@ -788,6 +797,7 @@ storage.provider.db.microchunkTree data:bytes = storage.provider.db.MicrochunkTr storage.daemon.queryError message:string = storage.daemon.QueryError; storage.daemon.success = storage.daemon.Success; + storage.daemon.torrent hash:int256 flags:# // 0 - info ready @@ -796,19 +806,29 @@ storage.daemon.torrent total_size:flags.0?long description:flags.0?string files_count:flags.1?long included_size:flags.1?long dir_name:flags.1?string downloaded_size:long - root_dir:string active_download:Bool active_upload:Bool completed:Bool + added_at:int root_dir:string active_download:Bool active_upload:Bool completed:Bool download_speed:double upload_speed:double fatal_error:flags.2?string = storage.daemon.Torrent; + storage.daemon.fileInfo - name:string size:long + name:string size:long flags:# priority:int downloaded_size:long = storage.daemon.FileInfo; + storage.daemon.torrentFull torrent:storage.daemon.torrent files:(vector storage.daemon.fileInfo) = storage.daemon.TorrentFull; storage.daemon.torrentList torrents:(vector storage.daemon.torrent) = storage.daemon.TorrentList; storage.daemon.torrentMeta meta:bytes = storage.daemon.TorrentMeta; +storage.daemon.filePiecesInfo name:string range_l:long range_r:long = storage.daemon.FilePiecesInfo; +storage.daemon.torrentPiecesInfo + flags:# // 0 - with file ranges + total_pieces:long piece_size:int + range_l:long range_r:long piece_ready_bitset:bytes + files:flags.0?(vector storage.daemon.filePiecesInfo) // files[0] is header + = storage.daemon.TorrentPiecesInfo; + storage.daemon.newContractParams rate:string max_span:int = storage.daemon.NewContractParams; storage.daemon.newContractParamsAuto provider_address:string = storage.daemon.NewContractParams; storage.daemon.newContractMessage body:bytes rate:string max_span:int = storage.daemon.NewContractMessage; @@ -821,6 +841,8 @@ storage.daemon.priorityPending = storage.daemon.SetPriorityStatus; storage.daemon.keyHash key_hash:int256 = storage.daemon.KeyHash; +storage.daemon.speedLimits download:double upload:double = storage.daemon.SpeedLimits; + storage.daemon.providerConfig max_contracts:int max_total_size:long = storage.daemon.ProviderConfig; storage.daemon.contractInfo address:string state:int torrent:int256 created_time:int file_size:long downloaded_size:long rate:string max_span:int client_balance:string contract_balance:string = storage.daemon.ContractInfo; @@ -831,24 +853,32 @@ storage.daemon.providerAddress address:string = storage.daemon.ProviderAddress; ---functions--- storage.daemon.setVerbosity verbosity:int = storage.daemon.Success; -storage.daemon.createTorrent path:string description:string allow_upload:Bool copy_inside:Bool = storage.daemon.TorrentFull; -storage.daemon.addByHash hash:int256 root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull; -storage.daemon.addByMeta meta:bytes root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull; +storage.daemon.createTorrent path:string description:string allow_upload:Bool copy_inside:Bool flags:# = storage.daemon.TorrentFull; +storage.daemon.addByHash hash:int256 root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) flags:# = storage.daemon.TorrentFull; +storage.daemon.addByMeta meta:bytes root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) flags:# = storage.daemon.TorrentFull; storage.daemon.setActiveDownload hash:int256 active:Bool = storage.daemon.Success; storage.daemon.setActiveUpload hash:int256 active:Bool = storage.daemon.Success; -storage.daemon.getTorrents = storage.daemon.TorrentList; -storage.daemon.getTorrentFull hash:int256 = storage.daemon.TorrentFull; -storage.daemon.getTorrentMeta hash:int256 = storage.daemon.TorrentMeta; +storage.daemon.getTorrents flags:# = storage.daemon.TorrentList; +storage.daemon.getTorrentFull hash:int256 flags:# = storage.daemon.TorrentFull; +storage.daemon.getTorrentMeta hash:int256 flags:# = storage.daemon.TorrentMeta; storage.daemon.getNewContractMessage hash:int256 query_id:long params:storage.daemon.NewContractParams = storage.daemon.NewContractMessage; -storage.daemon.getTorrentPeers hash:int256 = storage.daemon.PeerList; +storage.daemon.getTorrentPeers hash:int256 flags:# = storage.daemon.PeerList; +storage.daemon.getTorrentPiecesInfo hash:int256 + flags:# // 0 - with file ranges + offset:long max_pieces:long + = storage.daemon.TorrentPiecesInfo; storage.daemon.setFilePriorityAll hash:int256 priority:int = storage.daemon.SetPriorityStatus; storage.daemon.setFilePriorityByIdx hash:int256 idx:long priority:int = storage.daemon.SetPriorityStatus; storage.daemon.setFilePriorityByName hash:int256 name:string priority:int = storage.daemon.SetPriorityStatus; storage.daemon.removeTorrent hash:int256 remove_files:Bool = storage.daemon.Success; -storage.daemon.loadFrom hash:int256 meta:bytes path:string = storage.daemon.Torrent; +storage.daemon.loadFrom hash:int256 meta:bytes path:string flags:# = storage.daemon.Torrent; + +storage.daemon.getSpeedLimits flags:# = storage.daemon.SpeedLimits; +storage.daemon.setSpeedLimits flags:# download:flags.0?double upload:flags.1?double = storage.daemon.Success; + storage.daemon.importPrivateKey key:PrivateKey = storage.daemon.KeyHash; storage.daemon.initProvider account_address:string = storage.daemon.Success; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 3c81ca06..8ce7a579 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index c9a7df3d..bcfc625d 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -52,7 +52,7 @@ ton.blockIdExt workchain:int32 shard:int64 seqno:int32 root_hash:bytes file_hash raw.fullAccountState balance:int64 code:bytes data:bytes last_transaction_id:internal.transactionId block_id:ton.blockIdExt frozen_hash:bytes sync_utime:int53 = raw.FullAccountState; raw.message source:accountAddress destination:accountAddress value:int64 fwd_fee:int64 ihr_fee:int64 created_lt:int64 body_hash:bytes msg_data:msg.Data = raw.Message; -raw.transaction utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 storage_fee:int64 other_fee:int64 in_msg:raw.message out_msgs:vector = raw.Transaction; +raw.transaction address:accountAddress utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 storage_fee:int64 other_fee:int64 in_msg:raw.message out_msgs:vector = raw.Transaction; raw.transactions transactions:vector previous_transaction_id:internal.transactionId = raw.Transactions; raw.extMessageInfo hash:bytes = raw.ExtMessageInfo; @@ -215,6 +215,7 @@ blocks.shards shards:vector = blocks.Shards; blocks.accountTransactionId account:bytes lt:int64 = blocks.AccountTransactionId; blocks.shortTxId mode:# account:mode.0?bytes lt:mode.1?int64 hash:mode.2?bytes = liteServer.TransactionId; blocks.transactions id:ton.blockIdExt req_count:int32 incomplete:Bool transactions:vector = blocks.Transactions; +blocks.transactionsExt id:ton.blockIdExt req_count:int32 incomplete:Bool transactions:vector = blocks.TransactionsExt; blocks.header id:ton.blockIdExt global_id:int32 version:int32 flags:# after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 gen_utime:int53 vert_seqno:# prev_blocks:vector = blocks.Header; //blocks.shortData header:blocks.Header transactions:blocks.Header = blocks.BlockData; @@ -319,6 +320,7 @@ blocks.getMasterchainInfo = blocks.MasterchainInfo; blocks.getShards id:ton.blockIdExt = blocks.Shards; blocks.lookupBlock mode:int32 id:ton.blockId lt:int64 utime:int32 = ton.BlockIdExt; blocks.getTransactions id:ton.blockIdExt mode:# count:# after:blocks.accountTransactionId = blocks.Transactions; +blocks.getTransactionsExt id:ton.blockIdExt mode:# count:# after:blocks.accountTransactionId = blocks.TransactionsExt; blocks.getBlockHeader id:ton.blockIdExt = blocks.Header; blocks.getMasterchainBlockSignatures seqno:int32 = blocks.BlockSignatures; blocks.getShardBlockProof id:ton.blockIdExt mode:# from:mode.0?ton.blockIdExt = blocks.ShardBlockProof; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index ffe4b1e7..023a4953 100644 Binary files a/tl/generate/scheme/tonlib_api.tlo and b/tl/generate/scheme/tonlib_api.tlo differ diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index d56bcd1a..a2dfa9cf 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -2594,9 +2594,9 @@ const MasterConfig& get_default_master_config() { "file_hash": "XplPz01CXAps5qeSWUtxcyBfdAo5zVb1N979KLSKD24=" }, "init_block" : { - "root_hash": "irEt9whDfgaYwD+8AzBlYzrMZHhrkhSVp3PU1s4DOz4=", - "seqno": 10171687, - "file_hash": "lay/bUKUUFDJXU9S6gx9GACQFl+uK+zX8SqHWS9oLZc=", + "root_hash": "YRkrcmZMvLBvjanwKCyL3w4oceGPtFfgx8ym1QKCK/4=", + "seqno": 27747086, + "file_hash": "N42xzPnJjDlE3hxPXOb+pNzXomgRtpX5AZzMPnIA41s=", "workchain": -1, "shard": -9223372036854775808 }, @@ -2966,6 +2966,7 @@ struct ToRawTransactions { std::vector> out_msgs; td::int64 fees = 0; td::int64 storage_fee = 0; + td::string address; if (info.transaction.not_null()) { data = to_bytes(info.transaction); block::gen::Transaction::Record trans; @@ -2974,11 +2975,6 @@ struct ToRawTransactions { } TRY_RESULT_ASSIGN(fees, to_balance(trans.total_fees)); - //LOG(ERROR) << fees; - - //std::ostringstream outp; - //block::gen::t_Transaction.print_ref(outp, info.transaction); - //LOG(INFO) << outp.str(); auto is_just = trans.r1.in_msg->prefetch_long(1); if (is_just == trans.r1.in_msg->fetch_long_eof) { @@ -3004,8 +3000,11 @@ struct ToRawTransactions { return td::Status::Error("Failed to fetch storage fee from transaction"); } storage_fee = storage_fees->to_long(); + auto std_address = block::StdAddress(info.blkid.id.workchain, trans.account_addr); + address = std_address.rserialize(true); } return tonlib_api::make_object( + tonlib_api::make_object(std::move(address)), info.now, data, tonlib_api::make_object(info.prev_trans_lt, info.prev_trans_hash.as_slice().str()), @@ -3032,6 +3031,74 @@ struct ToRawTransactions { return tonlib_api::make_object(std::move(transactions), std::move(transaction_id)); } + + td::Result> to_raw_transaction_or_throw( + block::BlockTransaction::Info&& info) { + std::string data; + + tonlib_api::object_ptr in_msg; + std::vector> out_msgs; + td::int64 fees = 0; + td::int64 storage_fee = 0; + td::string address; + if (info.transaction.not_null()) { + data = to_bytes(info.transaction); + block::gen::Transaction::Record trans; + if (!tlb::unpack_cell(info.transaction, trans)) { + return td::Status::Error("Failed to unpack Transaction"); + } + + TRY_RESULT_ASSIGN(fees, to_balance(trans.total_fees)); + + auto is_just = trans.r1.in_msg->prefetch_long(1); + if (is_just == trans.r1.in_msg->fetch_long_eof) { + return td::Status::Error("Failed to parse long"); + } + if (is_just == -1) { + auto msg = trans.r1.in_msg->prefetch_ref(); + TRY_RESULT(in_msg_copy, to_raw_message(trans.r1.in_msg->prefetch_ref())); + in_msg = std::move(in_msg_copy); + } + + if (trans.outmsg_cnt != 0) { + vm::Dictionary dict{trans.r1.out_msgs, 15}; + for (int x = 0; x < trans.outmsg_cnt; x++) { + TRY_RESULT(out_msg, to_raw_message(dict.lookup_ref(td::BitArray<15>{x}))); + fees += out_msg->fwd_fee_; + fees += out_msg->ihr_fee_; + out_msgs.push_back(std::move(out_msg)); + } + } + td::RefInt256 storage_fees; + if (!block::tlb::t_TransactionDescr.get_storage_fees(trans.description, storage_fees)) { + return td::Status::Error("Failed to fetch storage fee from transaction"); + } + storage_fee = storage_fees->to_long(); + auto std_address = block::StdAddress(info.blkid.id.workchain, trans.account_addr); + address = std_address.rserialize(true); + } + return tonlib_api::make_object( + tonlib_api::make_object(std::move(address)), + info.now, data, + tonlib_api::make_object(info.lt, + info.hash.as_slice().str()), + fees, storage_fee, fees - storage_fee, std::move(in_msg), std::move(out_msgs)); + } + + td::Result> to_raw_transaction(block::BlockTransaction::Info&& info) { + return TRY_VM(to_raw_transaction_or_throw(std::move(info))); + } + + td::Result>> to_raw_transactions( + block::BlockTransactionList::Info&& info) { + std::vector> transactions; + for (auto& transaction : info.transactions) { + TRY_RESULT(raw_transaction, to_raw_transaction(std::move(transaction))); + transactions.push_back(std::move(raw_transaction)); + } + + return std::move(transactions); + } }; // Raw @@ -5215,25 +5282,116 @@ auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactions& request, td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) - TRY_RESULT(account, to_bits256((*request.after_).account_, "account")); - auto after = ton::lite_api::make_object(account, (*request.after_).lt_); + auto root_hash = block->root_hash_; + bool check_proof = request.mode_ & 32; + bool reverse_mode = request.mode_ & 64; + bool has_starting_tx = request.mode_ & 128; + + td::Bits256 start_addr; + ton::LogicalTime start_lt; + ton::lite_api::object_ptr after; + if (has_starting_tx) { + if (!request.after_) { + return td::Status::Error("Missing field `after`"); + } + TRY_RESULT_ASSIGN(start_addr, to_bits256(request.after_->account_, "account")); + start_lt = request.after_->lt_; + after = ton::lite_api::make_object(start_addr, start_lt); + } else { + start_addr = reverse_mode ? td::Bits256::ones() : td::Bits256::zero(); + start_lt = reverse_mode ? ~0ULL : 0; + after = nullptr; + } + client_.send_query(ton::lite_api::liteServer_listBlockTransactions( std::move(block), request.mode_, request.count_, std::move(after), - false, - false), - promise.wrap([](lite_api_ptr&& bTxes) { - const auto& id = bTxes->id_; - //for (auto id : ids) { + reverse_mode, + check_proof), + promise.wrap([check_proof, reverse_mode, root_hash, req_count = request.count_, start_addr, start_lt, mode = request.mode_] + (lite_api_ptr&& bTxes) -> td::Result> { + if (check_proof) { + try { + constexpr int max_answer_transactions = 256; + TRY_RESULT(proof_cell, vm::std_boc_deserialize(std::move(bTxes->proof_))); + auto virt_root = vm::MerkleProof::virtualize(proof_cell, 1); + + if (root_hash != virt_root->get_hash().bits()) { + return td::Status::Error("Invalid block proof root hash"); + } + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(std::move(blk.extra), extra))) { + return td::Status::Error("Error unpacking proof cell"); + } + vm::AugmentedDictionary acc_dict{vm::load_cell_slice_ref(extra.account_blocks), 256, + block::tlb::aug_ShardAccountBlocks}; + + bool eof = false; + ton::LogicalTime reverse = reverse_mode ? ~0ULL : 0; + ton::LogicalTime trans_lt = static_cast(start_lt); + td::Bits256 cur_addr = start_addr; + bool allow_same = true; + int count = 0; + while (!eof && count < req_count && count < max_answer_transactions) { + auto value = acc_dict.extract_value( + acc_dict.vm::DictionaryFixed::lookup_nearest_key(cur_addr.bits(), 256, !reverse, allow_same)); + if (value.is_null()) { + eof = true; + break; + } + allow_same = false; + if (cur_addr != start_addr) { + trans_lt = reverse; + } + + block::gen::AccountBlock::Record acc_blk; + if (!tlb::csr_unpack(std::move(value), acc_blk) || acc_blk.account_addr != cur_addr) { + return td::Status::Error("Error unpacking proof account block"); + } + vm::AugmentedDictionary trans_dict{vm::DictNonEmpty(), std::move(acc_blk.transactions), 64, + block::tlb::aug_AccountTransactions}; + td::BitArray<64> cur_trans{(long long)trans_lt}; + while (count < req_count && count < max_answer_transactions) { + auto tvalue = trans_dict.extract_value_ref( + trans_dict.vm::DictionaryFixed::lookup_nearest_key(cur_trans.bits(), 64, !reverse)); + if (tvalue.is_null()) { + trans_lt = reverse; + break; + } + if (static_cast(count) < bTxes->ids_.size()) { + if (mode & 4 && !tvalue->get_hash().bits().equals(bTxes->ids_[count]->hash_.bits(), 256)) { + return td::Status::Error("Couldn't verify proof (hash)"); + } + if (mode & 2 && cur_trans != td::BitArray<64>(bTxes->ids_[count]->lt_)) { + return td::Status::Error("Couldn't verify proof (lt)"); + } + if (mode & 1 && cur_addr != bTxes->ids_[count]->account_) { + return td::Status::Error("Couldn't verify proof (account)"); + } + } + count++; + } + } + if (static_cast(count) != bTxes->ids_.size()) { + return td::Status::Error(PSLICE() << "Txs count mismatch in proof (" << count << ") and response (" << bTxes->ids_.size() << ")"); + } + } catch (vm::VmError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (vm::VmVirtError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (...) { + return td::Status::Error("Unknown exception raised while verifying proof"); + } + } + tonlib_api::blocks_transactions r; - r.id_ = to_tonlib_api(*id); + r.id_ = to_tonlib_api(*bTxes->id_); r.req_count_ = bTxes->req_count_; r.incomplete_ = bTxes->incomplete_; for (auto& id: bTxes->ids_) { - //tonlib_api::blocks_shortTxId txid = tonlib_api::blocks_shortTxId(id->mode_, id->account_.as_slice().str(), id->lt_, id->hash_.as_slice().str()); - //r.transactions_.push_back(txid); r.transactions_.push_back(to_tonlib_api(*id)); } return tonlib_api::make_object(std::move(r)); @@ -5241,6 +5399,70 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactions& re return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactionsExt& request, + td::Promise>&& promise) { + TRY_RESULT(block, to_lite_api(*request.id_)) + bool check_proof = request.mode_ & 32; + bool reverse_mode = request.mode_ & 64; + bool has_starting_tx = request.mode_ & 128; + + td::Bits256 start_addr; + ton::LogicalTime start_lt; + ton::lite_api::object_ptr after; + if (has_starting_tx) { + if (!request.after_) { + return td::Status::Error("Missing field `after`"); + } + TRY_RESULT_ASSIGN(start_addr, to_bits256(request.after_->account_, "account")); + start_lt = request.after_->lt_; + after = ton::lite_api::make_object(start_addr, start_lt); + } else { + start_addr = reverse_mode ? td::Bits256::ones() : td::Bits256::zero(); + start_lt = reverse_mode ? ~0ULL : 0; + after = nullptr; + } + auto block_id = ton::create_block_id(block); + client_.send_query(ton::lite_api::liteServer_listBlockTransactionsExt( + std::move(block), + request.mode_, + request.count_, + std::move(after), + reverse_mode, + check_proof), + promise.wrap([block_id, check_proof, reverse_mode, start_addr, start_lt, req_count = request.count_] + (lite_api_ptr&& bTxes) -> td::Result> { + if (block_id != create_block_id(bTxes->id_)) { + return td::Status::Error("Liteserver responded with wrong block"); + } + + block::BlockTransactionList list; + list.blkid = block_id; + list.transactions_boc = std::move(bTxes->transactions_); + list.proof_boc = std::move(bTxes->proof_); + list.reverse_mode = reverse_mode; + list.start_lt = start_lt; + list.start_addr = start_addr; + list.req_count = req_count; + auto info = list.validate(check_proof); + if (info.is_error()) { + return info.move_as_error_prefix("Validation of block::BlockTransactionList failed: "); + } + + auto raw_transactions = ToRawTransactions(td::optional()).to_raw_transactions(info.move_as_ok()); + if (raw_transactions.is_error()) { + return raw_transactions.move_as_error_prefix("Error occured while creating tonlib_api::raw_transaction: "); + } + + tonlib_api::blocks_transactionsExt r; + r.id_ = to_tonlib_api(*bTxes->id_); + r.req_count_ = bTxes->req_count_; + r.incomplete_ = bTxes->incomplete_; + r.transactions_ = raw_transactions.move_as_ok(); + return tonlib_api::make_object(std::move(r)); + })); + return td::Status::OK(); +} + td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& request, td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index dbbb62a2..ed81760f 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -375,6 +375,8 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); td::Status do_request(const tonlib_api::blocks_getTransactions& block_data, td::Promise>&& promise); + td::Status do_request(const tonlib_api::blocks_getTransactionsExt& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::blocks_getBlockHeader& request, td::Promise>&& promise); td::Status do_request(const tonlib_api::blocks_getMasterchainBlockSignatures& request, diff --git a/utils/generate-random-id.cpp b/utils/generate-random-id.cpp index 3727f291..f606f358 100644 --- a/utils/generate-random-id.cpp +++ b/utils/generate-random-id.cpp @@ -28,9 +28,6 @@ #include #include #include -#include -#include -#include "crypto/ellcurve/Ed25519.h" #include "adnl/utils.hpp" #include "auto/tl/ton_api.h" #include "auto/tl/ton_api_json.h" @@ -38,12 +35,13 @@ #include "td/utils/OptionParser.h" #include "td/utils/filesystem.h" #include "keys/encryptor.h" -#include "keys/keys.hpp" #include "git.h" +#include "dht/dht-node.hpp" int main(int argc, char *argv[]) { ton::PrivateKey pk; - ton::tl_object_ptr addr_list; + td::optional addr_list; + td::optional network_id_opt; td::OptionParser p; p.set_description("generate random id"); @@ -78,11 +76,19 @@ int main(int argc, char *argv[]) { if (addr_list) { return td::Status::Error("duplicate '-a' option"); } - CHECK(!addr_list); td::BufferSlice bs(key); TRY_RESULT_PREFIX(as_json_value, td::json_decode(bs.as_slice()), "bad addr list JSON: "); - TRY_STATUS_PREFIX(td::from_json(addr_list, std::move(as_json_value)), "bad addr list TL: "); + ton::tl_object_ptr addr_list_tl; + TRY_STATUS_PREFIX(td::from_json(addr_list_tl, std::move(as_json_value)), "bad addr list TL: "); + TRY_RESULT_PREFIX_ASSIGN(addr_list, ton::adnl::AdnlAddressList::create(addr_list_tl), "bad addr list: "); + return td::Status::OK(); + }); + p.add_checked_option('i', "network-id", "dht network id (default: -1)", [&](td::Slice key) { + if (network_id_opt) { + return td::Status::Error("duplicate '-i' option"); + } + TRY_RESULT_PREFIX_ASSIGN(network_id_opt, td::to_integer_safe(key), "bad network id: "); return td::Status::OK(); }); @@ -118,7 +124,7 @@ int main(int argc, char *argv[]) { std::cerr << "'-a' option missing" << std::endl; return 2; } - auto x = ton::create_tl_object(pub_key.tl(), std::move(addr_list)); + auto x = ton::create_tl_object(pub_key.tl(), addr_list.value().tl()); auto e = pk.create_decryptor().move_as_ok(); auto r = e->sign(ton::serialize_tl_object(x, true).as_slice()).move_as_ok(); @@ -129,12 +135,17 @@ int main(int argc, char *argv[]) { std::cerr << "'-a' option missing" << std::endl; return 2; } - auto x = ton::create_tl_object(pub_key.tl(), std::move(addr_list), -1, td::BufferSlice()); + td::int32 network_id = network_id_opt ? network_id_opt.value() : -1; + td::BufferSlice to_sign = ton::serialize_tl_object( + ton::dht::DhtNode{ton::adnl::AdnlNodeIdFull{pub_key}, addr_list.value(), -1, network_id, td::BufferSlice{}} + .tl(), + true); auto e = pk.create_decryptor().move_as_ok(); - auto r = e->sign(ton::serialize_tl_object(x, true).as_slice()).move_as_ok(); - x->signature_ = std::move(r); + auto signature = e->sign(to_sign.as_slice()).move_as_ok(); + auto node = + ton::dht::DhtNode{ton::adnl::AdnlNodeIdFull{pub_key}, addr_list.value(), -1, network_id, std::move(signature)}; - auto v = td::json_encode(td::ToJson(x)); + auto v = td::json_encode(td::ToJson(node.tl())); std::cout << v << "\n"; } else if (mode == "keys") { td::write_file(name, pk.export_as_slice()).ensure(); diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 1aee7cc4..bd13225a 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -32,6 +32,7 @@ #include "terminal/terminal.h" #include "td/utils/filesystem.h" #include "overlay/overlays.h" +#include "ton/ton-tl.hpp" #include #include @@ -1055,3 +1056,54 @@ td::Status GetPerfTimerStatsJsonQuery::receive(td::BufferSlice data) { td::TerminalIO::output(std::string("wrote stats to " + file_name_ + "\n")); return td::Status::OK(); } + +td::Status GetShardOutQueueSizeQuery::run() { + TRY_RESULT_ASSIGN(block_id_.workchain, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(block_id_.shard, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(block_id_.seqno, tokenizer_.get_token()); + if (!tokenizer_.endl()) { + ton::ShardIdFull dest; + TRY_RESULT_ASSIGN(dest.workchain, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(dest.shard, tokenizer_.get_token()); + dest_ = dest; + } + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetShardOutQueueSizeQuery::send() { + auto b = ton::create_serialize_tl_object( + dest_ ? 1 : 0, ton::create_tl_block_id_simple(block_id_), dest_ ? dest_.value().workchain : 0, + dest_ ? dest_.value().shard : 0); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetShardOutQueueSizeQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "Queue_size: " << f->size_ << "\n"; + return td::Status::OK(); +} + +td::Status SetExtMessagesBroadcastDisabledQuery::run() { + TRY_RESULT(x, tokenizer_.get_token()); + if (x < 0 || x > 1) { + return td::Status::Error("value should be 0 or 1"); + } + value = x; + return td::Status::OK(); +} + +td::Status SetExtMessagesBroadcastDisabledQuery::send() { + auto b = ton::create_serialize_tl_object(value); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetExtMessagesBroadcastDisabledQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index ab2141dd..b1bdac7c 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -33,6 +33,7 @@ #include "td/utils/SharedSlice.h" #include "td/utils/port/IPAddress.h" #include "td/actor/actor.h" +#include "ton/ton-types.h" #include "keys/keys.hpp" @@ -1096,3 +1097,50 @@ class GetPerfTimerStatsJsonQuery : public Query { private: std::string file_name_; }; + +class GetShardOutQueueSizeQuery : public Query { + public: + GetShardOutQueueSizeQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getshardoutqueuesize"; + } + static std::string get_help() { + return "getshardoutqueuesize [ ]\treturns number of messages in the " + "queue of the given shard. Destination shard is optional."; + } + std::string name() const override { + return get_name(); + } + + private: + ton::BlockId block_id_; + td::optional dest_; +}; + +class SetExtMessagesBroadcastDisabledQuery : public Query { + public: + SetExtMessagesBroadcastDisabledQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "setextmessagesbroadcastdisabled"; + } + static std::string get_help() { + return "setextmessagesbroadcastdisabled \tdisable broadcasting and rebroadcasting ext messages; value is 0 " + "or 1."; + } + std::string name() const override { + return get_name(); + } + + private: + bool value; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 5ce8526b..01acced9 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -141,6 +141,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/CMakeLists.txt b/validator-engine/CMakeLists.txt index 6c1ea7e2..d369a2c3 100644 --- a/validator-engine/CMakeLists.txt +++ b/validator-engine/CMakeLists.txt @@ -12,7 +12,7 @@ set(VALIDATOR_ENGINE_SOURCE add_executable(validator-engine ${VALIDATOR_ENGINE_SOURCE}) target_link_libraries(validator-engine overlay tdutils tdactor adnl tl_api dht - rldp catchain validatorsession full-node validator ton_validator validator + rldp rldp2 catchain validatorsession full-node validator ton_validator validator fift-lib memprof git ${JEMALLOC_LIBRARIES}) install(TARGETS validator-engine RUNTIME DESTINATION bin) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index b6eb8b26..02ab42f9 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -69,7 +69,8 @@ #include #include #include "git.h" - +#include "block-auto.h" +#include "block-parse.h" Config::Config() { out_port = 3278; @@ -149,6 +150,10 @@ Config::Config(ton::ton_api::engine_validator_config &config) { config_add_full_node_master(s->port_, ton::PublicKeyHash{s->adnl_}).ensure(); } + if (config.fullnodeconfig_) { + full_node_config = ton::validator::fullnode::FullNodeConfig(config.fullnodeconfig_); + } + for (auto &serv : config.liteservers_) { config_add_lite_server(ton::PublicKeyHash{serv->id_}, serv->port_).ensure(); } @@ -219,6 +224,11 @@ ton::tl_object_ptr Config::tl() const { ton::create_tl_object(x.first, x.second.tl())); } + ton::tl_object_ptr full_node_config_obj = {}; + if (full_node_config != ton::validator::fullnode::FullNodeConfig()) { + full_node_config_obj = full_node_config.tl(); + } + std::vector> liteserver_vec; for (auto &x : liteservers) { liteserver_vec.push_back(ton::create_tl_object(x.second.tl(), x.first)); @@ -240,8 +250,8 @@ ton::tl_object_ptr Config::tl() const { } return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), full_node.tl(), - std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), - std::move(control_vec), std::move(gc_vec)); + std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(full_node_config_obj), + std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -1742,6 +1752,7 @@ void ValidatorEngine::started_dht() { void ValidatorEngine::start_rldp() { rldp_ = ton::rldp::Rldp::create(adnl_.get()); + rldp2_ = ton::rldp2::Rldp::create(adnl_.get()); started_rldp(); } @@ -1804,7 +1815,7 @@ void ValidatorEngine::start_full_node() { } full_node_ = ton::validator::fullnode::FullNode::create( short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, - keyring_.get(), adnl_.get(), rldp_.get(), + config_.full_node_config, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_); } @@ -3333,6 +3344,112 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTi td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, std::move(P)); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + + ton::BlockId block_id = ton::create_block_id_simple(query.block_id_); + if (!block_id.is_valid_ext()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "invalid block id"))); + return; + } + td::optional dest; + if (query.flags_ & 1) { + dest = ton::ShardIdFull{query.dest_wc_, (ton::ShardId)query.dest_shard_}; + if (!dest.value().is_valid_ext()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "invalid shard"))); + return; + } + } + + td::actor::send_closure( + validator_manager_, &ton::validator::ValidatorManagerInterface::get_block_by_seqno_from_db, + ton::AccountIdPrefixFull{block_id.workchain, block_id.shard}, block_id.seqno, + [=, promise = std::move(promise), + manager = validator_manager_.get()](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + auto handle = R.move_as_ok(); + if (handle->id().id != block_id) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "no such block"))); + return; + } + td::actor::send_closure( + manager, &ton::validator::ValidatorManagerInterface::get_shard_state_from_db, handle, + [=, promise = std::move(promise)](td::Result> R) mutable { + auto res = [&]() -> td::Result { + TRY_RESULT(state, std::move(R)); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { + return td::Status::Error(ton::ErrorCode::error, "invalid message queue"); + } + auto queue = std::make_unique(qinfo.out_queue->prefetch_ref(0), 352, + block::tlb::aug_OutMsgQueue); + if (dest) { + td::BitArray<96> prefix; + td::BitPtr ptr = prefix.bits(); + ptr.store_int(dest.value().workchain, 32); + ptr.advance(32); + ptr.store_uint(dest.value().shard, 64); + if (!queue->cut_prefix_subdict(prefix.bits(), 32 + dest.value().pfx_len())) { + return td::Status::Error(ton::ErrorCode::error, "invalid message queue"); + } + } + int size = 0; + queue->check_for_each([&](td::Ref, td::ConstBitPtr, int) -> bool { + ++size; + return true; + }); + return ton::create_serialize_tl_object(size); + }(); + if (res.is_error()) { + promise.set_value(create_control_query_error(res.move_as_error())); + } else { + promise.set_value(res.move_as_ok()); + } + }); + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setExtMessagesBroadcastDisabled &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + if (config_.full_node_config.ext_messages_broadcast_disabled_ == query.disabled_) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + config_.full_node_config.ext_messages_broadcast_disabled_ = query.disabled_; + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::set_config, config_.full_node_config); + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::create_serialize_tl_object()); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 7284a5be..e2367419 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -30,6 +30,7 @@ #include "adnl/adnl.h" #include "auto/tl/ton_api.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "dht/dht.h" #include "validator/manager.h" #include "validator/validator.h" @@ -85,6 +86,7 @@ struct Config { std::vector full_node_slaves; std::map full_node_masters; std::map liteservers; + ton::validator::fullnode::FullNodeConfig full_node_config; std::map controls; std::set gc; @@ -137,6 +139,7 @@ class ValidatorEngine : public td::actor::Actor { td::actor::ActorOwn adnl_network_manager_; td::actor::ActorOwn adnl_; td::actor::ActorOwn rldp_; + td::actor::ActorOwn rldp2_; std::map> dht_nodes_; ton::PublicKeyHash default_dht_node_ = ton::PublicKeyHash::zero(); td::actor::ActorOwn overlay_manager_; @@ -409,6 +412,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setExtMessagesBroadcastDisabled &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 3f56e3a3..2902b082 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -45,8 +45,8 @@ void ValidatorSessionImpl::process_blocks(std::vector std::vector> msgs; if (generated_ && !sent_generated_) { - auto it = blocks_[0].find(generated_block_); - CHECK(it != blocks_[0].end()); + auto it = blocks_.find(generated_block_); + CHECK(it != blocks_.end()); auto &B = it->second; auto file_hash = sha256_bits256(B->data_); @@ -230,13 +230,15 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice auto block_round = static_cast(candidate->round_); auto block_id = description().candidate_id(src_idx, candidate->root_hash_, file_hash, collated_data_file_hash); - if (block_round < cur_round_ || block_round >= cur_round_ + blocks_.size()) { + if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || + block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK) { VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id << "]: bad round=" << block_round << " cur_round" << cur_round_; return; } - auto it = blocks_[block_round - cur_round_].find(block_id); - if (it != blocks_[block_round - cur_round_].end()) { + auto it = blocks_.find(block_id); + if (it != blocks_.end()) { + it->second->round_ = std::max(it->second->round_, block_round); VLOG(VALIDATOR_SESSION_INFO) << this << "[node " << src << "][broadcast " << block_id << "]: duplicate"; return; } @@ -248,7 +250,7 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } - blocks_[block_round - cur_round_][block_id] = std::move(candidate); + blocks_[block_id] = std::move(candidate); VLOG(VALIDATOR_SESSION_WARNING) << this << ": received broadcast " << block_id; if (block_round != cur_round_) { @@ -407,7 +409,7 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(B)); - blocks_[0].emplace(block_id, std::move(b)); + blocks_.emplace(block_id, std::move(b)); pending_generate_ = false; generated_ = true; generated_block_ = block_id; @@ -507,9 +509,10 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { if (block) { auto T = td::Timestamp::at(round_started_at_.at() + description().get_delay(block->get_src_idx()) + 2.0); - auto it = blocks_[0].find(block_id); + auto it = blocks_.find(block_id); - if (it != blocks_[0].end()) { + if (it != blocks_.end()) { + it->second->round_ = std::max(it->second->round_, cur_round_); td::PerfWarningTimer timer{"too long block validation", 1.0}; auto &B = it->second; @@ -532,7 +535,6 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { } }); pending_approve_.insert(block_id); - CHECK(static_cast(cur_round_) == B->round_); callback_->on_candidate(cur_round_, description().get_source_public_key(block->get_src_idx()), B->root_hash_, B->data_.clone(), B->collated_data_.clone(), std::move(P)); @@ -556,7 +558,7 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { get_broadcast_p2p(id, block->get_file_hash(), block->get_collated_data_file_hash(), description().get_source_id(block->get_src_idx()), cur_round_, block->get_root_hash(), - std::move(P), td::Timestamp::in(2.0)); + std::move(P), td::Timestamp::in(15.0)); } else { LOG(VALIDATOR_SESSION_DEBUG) << this << ": no nodes to download candidate " << block << " from"; } @@ -773,7 +775,7 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { } } - auto it = blocks_[0].find(SentBlock::get_block_id(block)); + auto it = blocks_.find(SentBlock::get_block_id(block)); bool have_block = (bool)block; if (!have_block) { callback_->on_block_skipped(cur_round_); @@ -788,7 +790,7 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { cur_stats_.creator = description().get_source_id(block->get_src_idx()); cur_stats_.self = description().get_source_id(local_idx()); - if (it == blocks_[0].end()) { + if (it == blocks_.end()) { callback_->on_block_committed(cur_round_, description().get_source_public_key(block->get_src_idx()), block->get_root_hash(), block->get_file_hash(), td::BufferSlice(), std::move(export_sigs), std::move(export_approve_sigs), std::move(cur_stats_)); @@ -804,10 +806,14 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { } else { stats_add_round(); } - for (size_t i = 0; i < blocks_.size() - 1; i++) { - blocks_[i] = std::move(blocks_[i + 1]); + auto it2 = blocks_.begin(); + while (it2 != blocks_.end()) { + if (it2->second->round_ < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK) { + it2 = blocks_.erase(it2); + } else { + ++it2; + } } - blocks_[blocks_.size() - 1].clear(); } round_started_at_ = td::Timestamp::now(); diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 54bba33d..e8274554 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -73,7 +73,7 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionCandidateId signed_block_; td::BufferSlice signature_; - std::array>, 100> blocks_; + std::map> blocks_; catchain::CatChainSessionId unique_hash_; @@ -204,6 +204,8 @@ class ValidatorSessionImpl : public ValidatorSession { private: static const size_t MAX_REJECT_REASON_SIZE = 1024; + static const td::int32 MAX_FUTURE_ROUND_BLOCK = 100; + static const td::int32 MAX_PAST_ROUND_BLOCK = 20; }; } // namespace validatorsession diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 65d7b348..573cd8e5 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -198,5 +198,5 @@ target_link_libraries(validator-disk PRIVATE tdutils tdactor adnl rldp tl_api dh target_link_libraries(validator-hardfork PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_crypto ton_block ton_db) -target_link_libraries(full-node PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec +target_link_libraries(full-node PRIVATE tdutils tdactor adnl rldp rldp2 tl_api dht tdfec overlay catchain validatorsession ton_crypto ton_block ton_db) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index e142043e..3b2a34f3 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -33,11 +33,11 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { explicit CellDbAsyncExecutor(td::actor::ActorId cell_db) : cell_db_(std::move(cell_db)) { } - void execute_async(std::function f) { + void execute_async(std::function f) override { class Runner : public td::actor::Actor { public: explicit Runner(std::function f) : f_(std::move(f)) {} - void start_up() { + void start_up() override { f_(); stop(); } @@ -47,7 +47,7 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { td::actor::create_actor("executeasync", std::move(f)).release(); } - void execute_sync(std::function f) { + void execute_sync(std::function f) override { td::actor::send_closure(cell_db_, &CellDbBase::execute_sync, std::move(f)); } private: @@ -83,7 +83,6 @@ void CellDbIn::start_up() { set_block(empty, std::move(e)); cell_db_->commit_write_batch().ensure(); } - last_gc_ = empty; } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { @@ -141,17 +140,15 @@ void CellDbIn::get_cell_db_reader(td::Promise> } void CellDbIn::alarm() { - auto R = get_block(last_gc_); - R.ensure(); - - auto N = R.move_as_ok(); + auto E = get_block(get_empty_key_hash()).move_as_ok(); + auto N = get_block(E.next).move_as_ok(); if (N.is_empty()) { - last_gc_ = N.next; alarm_timestamp() = td::Timestamp::in(0.1); return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto block_id = N.block_id; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &CellDbIn::skip_gc); } else { @@ -159,24 +156,19 @@ void CellDbIn::alarm() { if (!value) { td::actor::send_closure(SelfId, &CellDbIn::skip_gc); } else { - td::actor::send_closure(SelfId, &CellDbIn::gc); + td::actor::send_closure(SelfId, &CellDbIn::gc, block_id); } } }); - td::actor::send_closure(root_db_, &RootDb::allow_state_gc, N.block_id, std::move(P)); + td::actor::send_closure(root_db_, &RootDb::allow_state_gc, block_id, std::move(P)); } -void CellDbIn::gc() { - auto R = get_block(last_gc_); - R.ensure(); - - auto N = R.move_as_ok(); - +void CellDbIn::gc(BlockIdExt block_id) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &CellDbIn::gc_cont, R.move_as_ok()); }); - td::actor::send_closure(root_db_, &RootDb::get_block_handle_external, N.block_id, false, std::move(P)); + td::actor::send_closure(root_db_, &RootDb::get_block_handle_external, block_id, false, std::move(P)); } void CellDbIn::gc_cont(BlockHandle handle) { @@ -196,7 +188,8 @@ void CellDbIn::gc_cont(BlockHandle handle) { void CellDbIn::gc_cont2(BlockHandle handle) { td::PerfWarningTimer timer{"gccell", 0.1}; - auto FR = get_block(last_gc_); + auto key_hash = get_key_hash(handle->id()); + auto FR = get_block(key_hash); FR.ensure(); auto F = FR.move_as_ok(); @@ -218,10 +211,10 @@ void CellDbIn::gc_cont2(BlockHandle handle) { boc_->dec(cell); boc_->prepare_commit().ensure(); - vm::CellStorer stor{*cell_db_.get()}; + vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); boc_->commit(stor).ensure(); - cell_db_->erase(get_key(last_gc_)).ensure(); + cell_db_->erase(get_key(key_hash)).ensure(); set_block(F.prev, std::move(P)); set_block(F.next, std::move(N)); cell_db_->commit_write_batch().ensure(); @@ -230,16 +223,11 @@ void CellDbIn::gc_cont2(BlockHandle handle) { boc_->set_loader(std::make_unique(cell_db_->snapshot())).ensure(); td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); - DCHECK(get_block(last_gc_).is_error()); - last_gc_ = F.next; + DCHECK(get_block(key_hash).is_error()); } void CellDbIn::skip_gc() { - auto FR = get_block(last_gc_); - FR.ensure(); - auto F = FR.move_as_ok(); - last_gc_ = F.next; - alarm_timestamp() = td::Timestamp::in(0.01); + alarm_timestamp() = td::Timestamp::in(1.0); } std::string CellDbIn::get_key(KeyHash key_hash) { @@ -296,7 +284,7 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise } else { promise.set_result(R.move_as_ok()); } - }); + }); boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P)); } } diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index e54526b9..a05e9ddb 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -84,7 +84,7 @@ class CellDbIn : public CellDbBase { static BlockIdExt get_empty_key(); KeyHash get_empty_key_hash(); - void gc(); + void gc(BlockIdExt block_id); void gc_cont(BlockHandle handle); void gc_cont2(BlockHandle handle); void skip_gc(); @@ -96,8 +96,6 @@ class CellDbIn : public CellDbBase { std::unique_ptr boc_; std::shared_ptr cell_db_; - - KeyHash last_gc_; }; class CellDb : public CellDbBase { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 4cb5627d..0f495077 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -96,6 +96,7 @@ void FullNodeShardImpl::create_overlay() { std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, adnl_id_); if (cert_) { td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_); } @@ -108,15 +109,17 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad } auto q = B.move_as_ok(); + if (config_.ext_messages_broadcast_disabled_) { + promise.set_error(td::Status::Error("rebroadcasting external messages is disabled")); + promise = [manager = validator_manager_, message = q->message_->data_.clone()](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(manager, &ValidatorManagerInterface::new_external_message, std::move(message)); + } + }; + } td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_external_message, std::move(q->message_->data_), - [promise = std::move(promise)](td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_result(td::Unit()); - } - }); + promise.wrap([](td::Ref) { return td::Unit(); })); } void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { @@ -685,6 +688,9 @@ void FullNodeShardImpl::send_ihr_message(td::BufferSlice data) { } void FullNodeShardImpl::send_external_message(td::BufferSlice data) { + if (config_.ext_messages_broadcast_disabled_) { + return; + } if (!client_.empty()) { td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "send_ext_query", create_serialize_tl_object_suffix( @@ -768,9 +774,11 @@ void FullNodeShardImpl::download_zero_state(BlockIdExt id, td::uint32 priority, void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { + auto &b = choose_neighbour(); td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, masterchain_block_id, - adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), priority, timeout, - validator_manager_, rldp_, overlays_, adnl_, client_, std::move(promise)) + adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, + b.use_rldp2() ? (td::actor::ActorId)rldp2_ : rldp_, + overlays_, adnl_, client_, std::move(promise)) .release(); } @@ -805,8 +813,9 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::stri td::Promise promise) { auto &b = choose_neighbour(); td::actor::create_actor( - "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), timeout, - validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) + "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, validator_manager_, + b.use_rldp2() ? (td::actor::ActorId)rldp2_ : rldp_, overlays_, adnl_, client_, + create_neighbour_promise(b, std::move(promise))) .release(); } @@ -1100,8 +1109,9 @@ void FullNodeShardImpl::ping_neighbours() { } FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, + FileHash zero_state_file_hash, FullNodeConfig config, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client) @@ -1112,18 +1122,21 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) + , rldp2_(rldp2) , overlays_(overlays) , validator_manager_(validator_manager) - , client_(client) { + , client_(client) + , config_(config) { } td::actor::ActorOwn FullNodeShard::create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId validator_manager, td::actor::ActorId client) { - return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, keyring, - adnl, rldp, overlays, validator_manager, client); + FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client) { + return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, config, + keyring, adnl, rldp, rldp2, overlays, validator_manager, client); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index c1712baf..1b742fb9 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -36,6 +36,7 @@ class FullNodeShard : public td::actor::Actor { virtual ShardIdFull get_shard_full() const = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; + virtual void set_config(FullNodeConfig config) = 0; virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; @@ -68,9 +69,10 @@ class FullNodeShard : public td::actor::Actor { static td::actor::ActorOwn create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId validator_manager, td::actor::ActorId client); + FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index a2dd5cc4..dcf4c649 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -44,6 +44,10 @@ struct Neighbour { void query_failed(); void update_roundtrip(double t); + bool use_rldp2() const { + return std::make_pair(proto_version, capabilities) >= std::make_pair(2, 2); + } + static Neighbour zero; }; @@ -66,7 +70,7 @@ class FullNodeShardImpl : public FullNodeShard { return 2; } static constexpr td::uint64 proto_capabilities() { - return 1; + return 2; } static constexpr td::uint32 max_neighbours() { return 16; @@ -81,6 +85,10 @@ class FullNodeShardImpl : public FullNodeShard { void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; + void set_config(FullNodeConfig config) override { + config_ = config; + } + //td::Result fetch_block(td::BufferSlice data); void prevalidate_block(BlockIdExt block_id, td::BufferSlice data, td::BufferSlice proof, td::Promise promise); @@ -198,9 +206,9 @@ class FullNodeShardImpl : public FullNodeShard { } FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, td::actor::ActorId keyring, + FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId overlays, + td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client); @@ -220,6 +228,7 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; td::actor::ActorId client_; @@ -239,6 +248,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Timestamp reload_neighbours_at_; td::Timestamp ping_neighbours_at_; adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); + + FullNodeConfig config_; }; } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 6606f215..ebba50a0 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -20,6 +20,7 @@ #include "ton/ton-shard.h" #include "ton/ton-io.hpp" #include "td/actor/MultiPromise.h" +#include "full-node.h" namespace ton { @@ -110,6 +111,13 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise R) { R.ensure(); @@ -123,8 +131,8 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { void FullNodeImpl::add_shard(ShardIdFull shard) { while (true) { if (shards_.count(shard) == 0) { - shards_.emplace(shard, FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, keyring_, adnl_, - rldp_, overlays_, validator_manager_, client_)); + shards_.emplace(shard, FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, + adnl_, rldp_, rldp2_, overlays_, validator_manager_, client_)); if (all_validators_.size() > 0) { td::actor::send_closure(shards_[shard], &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } @@ -449,8 +457,9 @@ void FullNodeImpl::start_up() { } FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId dht, + FullNodeConfig config, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId rldp2, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, std::string db_root) @@ -460,24 +469,40 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) + , rldp2_(rldp2) , dht_(dht) , overlays_(overlays) , validator_manager_(validator_manager) , client_(client) - , db_root_(db_root) { + , db_root_(db_root) + , config_(config) { add_shard(ShardIdFull{masterchainId}); } td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, + FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, + td::actor::ActorId rldp2, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, std::string db_root) { - return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, keyring, adnl, rldp, - dht, overlays, validator_manager, client, db_root); + return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, config, keyring, + adnl, rldp, rldp2, dht, overlays, validator_manager, client, db_root); +} + +FullNodeConfig::FullNodeConfig(const tl_object_ptr &obj) + : ext_messages_broadcast_disabled_(obj->ext_messages_broadcast_disabled_) { +} + +tl_object_ptr FullNodeConfig::tl() const { + return create_tl_object(ext_messages_broadcast_disabled_); +} +bool FullNodeConfig::operator==(const FullNodeConfig &rhs) const { + return ext_messages_broadcast_disabled_ == rhs.ext_messages_broadcast_disabled_; +} +bool FullNodeConfig::operator!=(const FullNodeConfig &rhs) const { + return !(*this == rhs); } } // namespace fullnode diff --git a/validator/full-node.h b/validator/full-node.h index cdf39d6f..15d54b55 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -27,6 +27,7 @@ #include "adnl/adnl.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "dht/dht.h" #include "overlay/overlays.h" #include "validator/validator.h" @@ -44,6 +45,16 @@ constexpr int VERBOSITY_NAME(FULL_NODE_INFO) = verbosity_DEBUG; constexpr int VERBOSITY_NAME(FULL_NODE_DEBUG) = verbosity_DEBUG; constexpr int VERBOSITY_NAME(FULL_NODE_EXTRA_DEBUG) = verbosity_DEBUG + 1; +struct FullNodeConfig { + FullNodeConfig() = default; + FullNodeConfig(const tl_object_ptr& obj); + tl_object_ptr tl() const; + bool operator==(const FullNodeConfig& rhs) const; + bool operator!=(const FullNodeConfig& rhs) const; + + bool ext_messages_broadcast_disabled_ = false; +}; + class FullNode : public td::actor::Actor { public: virtual ~FullNode() = default; @@ -61,6 +72,7 @@ class FullNode : public td::actor::Actor { td::Promise promise) = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; + virtual void set_config(FullNodeConfig config) = 0; static constexpr td::uint32 max_block_size() { return 4 << 20; @@ -73,10 +85,10 @@ class FullNode : public td::actor::Actor { } static td::actor::ActorOwn create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, + FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, + td::actor::ActorId rldp2, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, std::string db_root); diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 6d57f4a8..fc2dd75c 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -49,8 +49,8 @@ class FullNodeImpl : public FullNode { std::shared_ptr cert, td::Promise promise) override; - void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; + void set_config(FullNodeConfig config) override; void add_shard(ShardIdFull shard); void del_shard(ShardIdFull shard); @@ -82,9 +82,9 @@ class FullNodeImpl : public FullNode { void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId dht, - td::actor::ActorId overlays, + FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, std::string db_root); @@ -101,6 +101,7 @@ class FullNodeImpl : public FullNode { td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; td::actor::ActorId dht_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; @@ -112,6 +113,7 @@ class FullNodeImpl : public FullNode { std::vector all_validators_; std::set local_keys_; + FullNodeConfig config_; }; } // namespace fullnode diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 66e73d8e..69861ba3 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -182,6 +182,11 @@ void LiteQuery::start_up() { (q.mode_ & 128) ? q.after_->account_ : td::Bits256::zero(), static_cast((q.mode_ & 128) ? (q.after_->lt_) : 0)); }, + [&](lite_api::liteServer_listBlockTransactionsExt& q) { + this->perform_listBlockTransactionsExt(ton::create_block_id(q.id_), q.mode_, q.count_, + (q.mode_ & 128) ? q.after_->account_ : td::Bits256::zero(), + static_cast((q.mode_ & 128) ? (q.after_->lt_) : 0)); + }, [&](lite_api::liteServer_getConfigParams& q) { this->perform_getConfigParams(ton::create_block_id(q.id_), (q.mode_ & 0xffff) | 0x10000, q.param_list_); }, @@ -1964,6 +1969,118 @@ void LiteQuery::finish_listBlockTransactions(int mode, int req_count) { finish_query(std::move(b)); } +void LiteQuery::perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int count, Bits256 account, LogicalTime lt) { + LOG(INFO) << "started a listBlockTransactionsExt(" << blkid.to_str() << ", " << mode << ", " << count << ", " + << account.to_hex() << ", " << lt << ") liteserver query"; + base_blk_id_ = blkid; + acc_addr_ = account; + trans_lt_ = lt; + set_continuation([this, mode, count]() -> void { finish_listBlockTransactionsExt(mode, count); }); + request_block_data(blkid); +} + +void LiteQuery::finish_listBlockTransactionsExt(int mode, int req_count) { + LOG(INFO) << "completing a listBlockTransactionsExt(" << base_blk_id_.to_str() << ", " << mode << ", " << req_count + << ", " << acc_addr_.to_hex() << ", " << trans_lt_ << ") liteserver query"; + constexpr int max_answer_transactions = 256; + CHECK(block_.not_null()); + auto block_root = block_->root_cell(); + CHECK(block_root.not_null()); + RootHash rhash{block_root->get_hash().bits()}; + CHECK(rhash == base_blk_id_.root_hash); + vm::MerkleProofBuilder pb; + auto virt_root = block_root; + if (mode & 32) { + // proof requested + virt_root = pb.init(std::move(virt_root)); + } + if ((mode & 192) == 64) { // reverse order, no starting point + acc_addr_.set_ones(); + trans_lt_ = ~0ULL; + } + std::vector> trans_roots; + bool eof = false; + ton::LogicalTime reverse = (mode & 64) ? ~0ULL : 0; + try { + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(std::move(blk.extra), extra))) { + fatal_error("cannot find account transaction data in block "s + base_blk_id_.to_str()); + return; + } + vm::AugmentedDictionary acc_dict{vm::load_cell_slice_ref(extra.account_blocks), 256, + block::tlb::aug_ShardAccountBlocks}; + int count = 0; + bool allow_same = true; + td::Bits256 cur_addr = acc_addr_; + while (!eof && count < req_count && count < max_answer_transactions) { + Ref value; + try { + value = acc_dict.extract_value( + acc_dict.vm::DictionaryFixed::lookup_nearest_key(cur_addr.bits(), 256, !reverse, allow_same)); + } catch (vm::VmError err) { + fatal_error("error while traversing account block dictionary: "s + err.get_msg()); + return; + } + if (value.is_null()) { + eof = true; + break; + } + allow_same = false; + if (cur_addr != acc_addr_) { + trans_lt_ = reverse; + } + block::gen::AccountBlock::Record acc_blk; + if (!(tlb::csr_unpack(std::move(value), acc_blk) && acc_blk.account_addr == cur_addr)) { + fatal_error("invalid AccountBlock for account "s + cur_addr.to_hex()); + return; + } + vm::AugmentedDictionary trans_dict{vm::DictNonEmpty(), std::move(acc_blk.transactions), 64, + block::tlb::aug_AccountTransactions}; + td::BitArray<64> cur_trans{(long long)trans_lt_}; + while (count < req_count && count < max_answer_transactions) { + Ref tvalue; + try { + tvalue = trans_dict.extract_value_ref( + trans_dict.vm::DictionaryFixed::lookup_nearest_key(cur_trans.bits(), 64, !reverse)); + } catch (vm::VmError err) { + fatal_error("error while traversing transaction dictionary of an AccountBlock: "s + err.get_msg()); + return; + } + if (tvalue.is_null()) { + trans_lt_ = reverse; + break; + } + trans_roots.push_back(std::move(tvalue)); + ++count; + } + } + } catch (vm::VmError err) { + fatal_error("error while parsing AccountBlocks of block "s + base_blk_id_.to_str() + " : " + err.get_msg()); + return; + } + td::BufferSlice proof_data; + if (mode & 32) { + // create proof + auto proof_boc = pb.extract_proof_boc(); + if (proof_boc.is_error()) { + fatal_error(proof_boc.move_as_error()); + return; + } + proof_data = proof_boc.move_as_ok(); + } + auto res = vm::std_boc_serialize_multi(std::move(trans_roots)); + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + + auto b = ton::create_serialize_tl_object( + ton::create_tl_lite_block_id(base_blk_id_), req_count, !eof, res.move_as_ok(), std::move(proof_data)); + LOG(INFO) << "listBlockTransactionsExt() query completed"; + finish_query(std::move(b)); +} + void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, int mode) { if (!(mode & 1)) { to.invalidate_clear(); diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 47970aae..2707fdfe 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -133,6 +133,8 @@ class LiteQuery : public td::actor::Actor { void perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, UnixTime utime); void perform_listBlockTransactions(BlockIdExt blkid, int mode, int count, Bits256 account, LogicalTime lt); void finish_listBlockTransactions(int mode, int count); + void perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int count, Bits256 account, LogicalTime lt); + void finish_listBlockTransactionsExt(int mode, int count); void perform_getBlockProof(BlockIdExt from, BlockIdExt to, int mode); void continue_getBlockProof(BlockIdExt from, BlockIdExt to, int mode, BlockIdExt baseblk, Ref state); diff --git a/validator/net/download-archive-slice.cpp b/validator/net/download-archive-slice.cpp index e3acb4ba..6235b8b0 100644 --- a/validator/net/download-archive-slice.cpp +++ b/validator/net/download-archive-slice.cpp @@ -29,7 +29,7 @@ namespace fullnode { DownloadArchiveSlice::DownloadArchiveSlice( BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, - td::actor::ActorId validator_manager, td::actor::ActorId rldp, + td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise) : masterchain_seqno_(masterchain_seqno) @@ -144,6 +144,8 @@ void DownloadArchiveSlice::got_archive_info(td::BufferSlice data) { return; } + prev_logged_timer_ = td::Timer(); + LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " from " << download_from_; get_archive_slice(); } @@ -159,12 +161,12 @@ void DownloadArchiveSlice::get_archive_slice() { auto q = create_serialize_tl_object(archive_id_, offset_, slice_size()); if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, - "get_archive_slice", std::move(P), td::Timestamp::in(3.0), std::move(q), + "get_archive_slice", std::move(P), td::Timestamp::in(15.0), std::move(q), slice_size() + 1024, rldp_); } else { td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_archive_slice", create_serialize_tl_object_suffix(std::move(q)), - td::Timestamp::in(1.0), std::move(P)); + td::Timestamp::in(15.0), std::move(P)); } } @@ -181,7 +183,16 @@ void DownloadArchiveSlice::got_archive_slice(td::BufferSlice data) { offset_ += data.size(); + double elapsed = prev_logged_timer_.elapsed(); + if (elapsed > 10.0) { + prev_logged_timer_ = td::Timer(); + LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << ": total=" << offset_ << " (" + << td::format::as_size((td::uint64)(double(offset_ - prev_logged_sum_) / elapsed)) << "/s)"; + prev_logged_sum_ = offset_; + } + if (data.size() < slice_size()) { + LOG(INFO) << "finished downloading arcrive slice #" << masterchain_seqno_ << ": total=" << offset_; finish_query(); } else { get_archive_slice(); diff --git a/validator/net/download-archive-slice.hpp b/validator/net/download-archive-slice.hpp index 6ef11d1a..0384ac8c 100644 --- a/validator/net/download-archive-slice.hpp +++ b/validator/net/download-archive-slice.hpp @@ -21,7 +21,6 @@ #include "overlay/overlays.h" #include "ton/ton-types.h" #include "validator/validator.h" -#include "rldp/rldp.h" #include "adnl/adnl-ext-client.h" #include "td/utils/port/FileFd.h" @@ -36,9 +35,9 @@ class DownloadArchiveSlice : public td::actor::Actor { DownloadArchiveSlice(BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, td::actor::ActorId validator_manager, - td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId adnl, td::actor::ActorId client, - td::Promise promise); + td::actor::ActorId rldp, + td::actor::ActorId overlays, td::actor::ActorId adnl, + td::actor::ActorId client, td::Promise promise); void abort_query(td::Status reason); void alarm() override; @@ -51,7 +50,7 @@ class DownloadArchiveSlice : public td::actor::Actor { void got_archive_slice(td::BufferSlice data); static constexpr td::uint32 slice_size() { - return 1 << 17; + return 1 << 21; } private: @@ -68,11 +67,14 @@ class DownloadArchiveSlice : public td::actor::Actor { td::Timestamp timeout_; td::actor::ActorId validator_manager_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId overlays_; td::actor::ActorId adnl_; td::actor::ActorId client_; td::Promise promise_; + + td::uint64 prev_logged_sum_ = 0; + td::Timer prev_logged_timer_; }; } // namespace fullnode diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp index ef5ed7e5..14754f64 100644 --- a/validator/net/download-block-new.cpp +++ b/validator/net/download-block-new.cpp @@ -201,12 +201,12 @@ void DownloadBlockNew::got_node_to_download(adnl::AdnlNodeIdShort node) { } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, - "get_proof", std::move(P), td::Timestamp::in(3.0), std::move(q), + "get_proof", std::move(P), td::Timestamp::in(15.0), std::move(q), FullNode::max_proof_size() + FullNode::max_block_size() + 128, rldp_); } else { td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_prepare", create_serialize_tl_object_suffix(std::move(q)), - td::Timestamp::in(1.0), std::move(P)); + td::Timestamp::in(15.0), std::move(P)); } } diff --git a/validator/net/download-block.cpp b/validator/net/download-block.cpp index 5e7c0be9..9ca84be2 100644 --- a/validator/net/download-block.cpp +++ b/validator/net/download-block.cpp @@ -373,12 +373,12 @@ void DownloadBlock::got_block_data_description(td::BufferSlice data_description) auto q = create_serialize_tl_object(create_tl_block_id(block_id_)); if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, - overlay_id_, "get_block", std::move(P), td::Timestamp::in(3.0), std::move(q), + overlay_id_, "get_block", std::move(P), td::Timestamp::in(15.0), std::move(q), FullNode::max_block_size(), rldp_); } else { td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_block", create_serialize_tl_object_suffix(std::move(q)), - td::Timestamp::in(3.0), std::move(P)); + td::Timestamp::in(15.0), std::move(P)); } }, [&](ton_api::tonNode_notFound &val) { diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index fedfaae8..2740ce41 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -32,9 +32,9 @@ DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_i overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::uint32 priority, td::Timestamp timeout, td::actor::ActorId validator_manager, - td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId adnl, td::actor::ActorId client, - td::Promise promise) + td::actor::ActorId rldp, + td::actor::ActorId overlays, td::actor::ActorId adnl, + td::actor::ActorId client, td::Promise promise) : block_id_(block_id) , masterchain_block_id_(masterchain_block_id) , local_id_(local_id) @@ -115,7 +115,7 @@ void DownloadState::got_block_handle(BlockHandle handle) { void DownloadState::got_node_to_download(adnl::AdnlNodeIdShort node) { download_from_ = node; - LOG(INFO) << "downloading state " << block_id_ << " from " << download_from_; + LOG(INFO) << "downloading state " << block_id_.to_str() << " from " << download_from_; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { if (R.is_error()) { @@ -192,8 +192,8 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques double elapsed = prev_logged_timer_.elapsed(); if (elapsed > 10.0) { prev_logged_timer_ = td::Timer(); - LOG(INFO) << "downloading state " << block_id_ << ": total=" << sum_ << - " (" << double(sum_ - prev_logged_sum_) / elapsed << " B/s)"; + LOG(INFO) << "downloading state " << block_id_.to_str() << ": total=" << sum_ << " (" + << td::format::as_size((td::uint64)(double(sum_ - prev_logged_sum_) / elapsed)) << "/s)"; prev_logged_sum_ = sum_; } @@ -210,7 +210,7 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques return; } - td::uint32 part_size = 1 << 18; + td::uint32 part_size = 1 << 21; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), part_size](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); @@ -223,18 +223,18 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), sum_, part_size); if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, - "download state", std::move(P), td::Timestamp::in(10.0), std::move(query), + "download state", std::move(P), td::Timestamp::in(20.0), std::move(query), FullNode::max_state_size(), rldp_); } else { td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "download state", create_serialize_tl_object_suffix(std::move(query)), - td::Timestamp::in(10.0), std::move(P)); + td::Timestamp::in(20.0), std::move(P)); } } void DownloadState::got_block_state(td::BufferSlice data) { state_ = std::move(data); - LOG(INFO) << "finished downloading state " << block_id_ << ": total=" << sum_; + LOG(INFO) << "finished downloading state " << block_id_.to_str() << ": total=" << sum_; finish_query(); } diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index a586f61f..7db1327f 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -21,7 +21,6 @@ #include "overlay/overlays.h" #include "ton/ton-types.h" #include "validator/validator.h" -#include "rldp/rldp.h" #include "adnl/adnl-ext-client.h" namespace ton { @@ -35,7 +34,7 @@ class DownloadState : public td::actor::Actor { DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::uint32 priority, td::Timestamp timeout, td::actor::ActorId validator_manager, - td::actor::ActorId rldp, td::actor::ActorId overlays, + td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise); @@ -62,7 +61,7 @@ class DownloadState : public td::actor::Actor { td::Timestamp timeout_; td::actor::ActorId validator_manager_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId overlays_; td::actor::ActorId adnl_; td::actor::ActorId client_; diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 1bb932c6..4ac4fc70 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -148,7 +148,7 @@ void AsyncStateSerializer::next_iteration() { running_ = true; delay_action( [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, - td::Timestamp::in(td::Random::fast(0, 4 * 3600))); + td::Timestamp::in(td::Random::fast(0, 1800))); return; } }