diff --git a/.github/script/fift-func-wasm-build-ubuntu.sh b/.github/script/fift-func-wasm-build-ubuntu.sh new file mode 100755 index 00000000..505ce137 --- /dev/null +++ b/.github/script/fift-func-wasm-build-ubuntu.sh @@ -0,0 +1,79 @@ +# The script build funcfift compiler to WASM + +# dependencies: +#sudo apt-get install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip nodejs + +export CC=$(which clang) +export CXX=$(which clang++) +export CCACHE_DISABLE=1 + +git clone https://github.com/openssl/openssl.git +cd openssl +git checkout OpenSSL_1_1_1j + +./config +make -j4 + +OPENSSL_DIR=`pwd` + +cd .. + +git clone https://github.com/madler/zlib.git +cd zlib +ZLIB_DIR=`pwd` + +cd .. + +# clone ton repo +git clone --recursive https://github.com/the-ton-tech/ton-blockchain.git + +# only to generate auto-block.cpp + +cd ton-blockchain +git pull +git checkout 1566a23b2bece49fd1de9ab2f35e88297d22829f +mkdir build +cd build +cmake -DCMAKE_BUILD_TYPE=Release -DZLIB_LIBRARY=/usr/lib/x86_64-linux-gnu/libz.so -DZLIB_INCLUDE_DIR=$ZLIB_DIR -DOPENSSL_ROOT_DIR=$OPENSSL_DIR -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so -DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.so .. +make -j4 fift + +rm -rf * + +cd ../.. + +git clone https://github.com/emscripten-core/emsdk.git +cd emsdk +./emsdk install latest +./emsdk activate latest +EMSDK_DIR=`pwd` + +source $EMSDK_DIR/emsdk_env.sh +export CC=$(which emcc) +export CXX=$(which em++) +export CCACHE_DISABLE=1 + +cd ../zlib + +emconfigure ./configure --static +emmake make -j4 +ZLIB_DIR=`pwd` + +cd ../openssl + +make clean +emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-ui +sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile +sed -i 's/-ldl//g' Makefile +sed -i 's/-O3/-Os/g' Makefile +emmake make depend +emmake make -j4 + +cd ../ton-blockchain + +cd build + +emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DZLIB_LIBRARY=$ZLIB_DIR/libz.a -DZLIB_INCLUDE_DIR=$ZLIB_DIR -DOPENSSL_ROOT_DIR=$OPENSSL_DIR -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a -DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.a -DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DCMAKE_CXX_FLAGS="-pthread -sUSE_ZLIB=1" .. + +cp -R ../crypto/smartcont ../crypto/fift/lib crypto + +emmake make -j4 funcfiftlib diff --git a/CMakeLists.txt b/CMakeLists.txt index 44dbae8b..75a89c41 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,7 +84,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_EXTENSIONS FALSE) #BEGIN internal +option(USE_EMSCRIPTEN "Use \"ON\" for config building wasm." OFF) option(TON_ONLY_TONLIB "Use \"ON\" to build only tonlib." OFF) +if (USE_EMSCRIPTEN) + set(TON_ONLY_TONLIB true) +endif() if (TON_ONLY_TONLIB) set(NOT_TON_ONLY_TONLIB false) else() @@ -197,10 +201,13 @@ find_package(Threads REQUIRED) find_package(ZLIB REQUIRED) if (TON_ARCH AND NOT MSVC) + CHECK_CXX_COMPILER_FLAG( "-march=${TON_ARCH}" COMPILER_OPT_ARCH_SUPPORTED ) if (TON_ARCH STREQUAL "apple-m1") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${TON_ARCH}") - else() + elseif(COMPILER_OPT_ARCH_SUPPORTED) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${TON_ARCH}") + elseif(NOT TON_ARCH STREQUAL "native") + message(FATAL_ERROR "Compiler doesn't support arch ${TON_ARCH}") endif() endif() if (THREADS_HAVE_PTHREAD_ARG) @@ -242,10 +249,14 @@ elseif (CLANG OR GCC) set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} -fvisibility=hidden -Wl,-dead_strip,-x,-S") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,--exclude-libs,ALL") + if (NOT USE_EMSCRIPTEN) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,--exclude-libs,ALL") + endif() set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") if (NOT TON_USE_ASAN AND NOT TON_USE_TSAN AND NOT MEMPROF) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--exclude-libs,ALL") + if (NOT USE_EMSCRIPTEN) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--exclude-libs,ALL") + endif() endif() endif() endif() diff --git a/README.md b/README.md index df8fefa8..177bc3ee 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,19 @@ +[![Stack Overflow Group][stack-overflow-badge]][stack-overflow-url] +[![Telegram Foundation Group][telegram-foundation-badge]][telegram-foundation-url] +[![Telegram Community Group][telegram-community-badge]][telegram-community-url] +[![Twitter Group][twitter-badge]][twitter-url] + +[telegram-foundation-badge]: https://img.shields.io/badge/-TON%20Foundation-2CA5E0?style=flat&logo=telegram&logoColor=white +[telegram-community-badge]: https://img.shields.io/badge/-TON%20Community-2CA5E0?style=flat&logo=telegram&logoColor=white +[telegram-foundation-url]: https://t.me/tonblockchain +[telegram-community-url]: https://t.me/toncoin +[twitter-badge]: https://img.shields.io/twitter/follow/ton_blockchain +[twitter-url]: https://twitter.com/ton_blockchain +[stack-overflow-badge]: https://img.shields.io/badge/-Stack%20Overflow-FE7A16?style=flat&logo=stack-overflow&logoColor=white +[stack-overflow-url]: https://stackoverflow.com/questions/tagged/ton + + + # TON Main TON monorepo, which includes the code of the node/validator, lite-client, tonlib, FunC compiler, etc. diff --git a/adnl/adnl-proxy.cpp b/adnl/adnl-proxy.cpp index 10171e0c..81ec8dcd 100644 --- a/adnl/adnl-proxy.cpp +++ b/adnl/adnl-proxy.cpp @@ -188,7 +188,6 @@ void Receiver::receive_from_client(td::IPAddress addr, td::BufferSlice data) { p.data = std::move(data); p.adnl_start_time = start_time(); p.seqno = out_seqno_; - p.data = std::move(data); auto enc = proxy_->encrypt(std::move(p)); diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 58b98088..c8c85370 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -179,6 +179,7 @@ set(FUNC_LIB_SOURCE func/stack-transform.cpp func/optimize.cpp func/codegen.cpp + func/func.cpp ) set(TLB_BLOCK_AUTO @@ -266,6 +267,8 @@ set(BIGINT_TEST_SOURCE PARENT_SCOPE ) +set(USE_EMSCRIPTEN ${USE_EMSCRIPTEN} PARENT_SCOPE) + add_library(ton_crypto STATIC ${TON_CRYPTO_SOURCE}) target_include_directories(ton_crypto PUBLIC $ @@ -305,13 +308,30 @@ target_include_directories(ton_block PUBLIC $ $) target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api) -add_executable(func func/func.cpp ${FUNC_LIB_SOURCE}) +add_executable(func func/func-main.cpp ${FUNC_LIB_SOURCE}) target_include_directories(func PUBLIC $) target_link_libraries(func PUBLIC ton_crypto src_parser git ton_block) if (WINGETOPT_FOUND) target_link_libraries_system(func wingetopt) endif() +if (USE_EMSCRIPTEN) + add_executable(funcfiftlib funcfiftlib/funcfiftlib.cpp ${FUNC_LIB_SOURCE}) + target_include_directories(funcfiftlib PUBLIC $) + target_link_libraries(funcfiftlib PUBLIC fift-lib src_parser git) + target_link_options(funcfiftlib PRIVATE -sEXPORTED_RUNTIME_METHODS=FS,ccall,cwrap,_malloc,free,UTF8ToString,stringToUTF8) + target_link_options(funcfiftlib PRIVATE -sEXPORTED_FUNCTIONS=_func_compile,_version) + target_link_options(funcfiftlib PRIVATE -sEXPORT_NAME=CompilerModule) + target_link_options(funcfiftlib PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0) + target_link_options(funcfiftlib PRIVATE -sFILESYSTEM=1) + target_link_options(funcfiftlib PRIVATE -Oz) + target_link_options(funcfiftlib PRIVATE -sIGNORE_MISSING_MAIN=1) + target_link_options(funcfiftlib PRIVATE -sAUTO_NATIVE_LIBRARIES=0) + target_link_options(funcfiftlib PRIVATE -sMODULARIZE=1) + target_link_options(funcfiftlib PRIVATE --embed-file ${CMAKE_CURRENT_SOURCE_DIR}/fift/lib@/fiftlib) + target_compile_options(funcfiftlib PRIVATE -sDISABLE_EXCEPTION_CATCHING=0) +endif() + add_executable(tlbc tl/tlbc.cpp) target_include_directories(tlbc PUBLIC $) target_link_libraries(tlbc PUBLIC ton_crypto src_parser) @@ -337,16 +357,26 @@ if (TON_USE_ASAN AND NOT WIN32) endif() file(MAKE_DIRECTORY smartcont/auto) -if (NOT CMAKE_CROSSCOMPILING) +if (NOT CMAKE_CROSSCOMPILING OR USE_EMSCRIPTEN) set(GENERATE_TLB_CMD tlbc) - add_custom_command( - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/block - COMMAND ${TURN_OFF_LSAN} - COMMAND ${GENERATE_TLB_CMD} -o block-auto -n block::gen -z block.tlb - COMMENT "Generate block tlb source files" - OUTPUT ${TLB_BLOCK_AUTO} - DEPENDS tlbc block/block.tlb - ) + if (NOT USE_EMSCRIPTEN) + add_custom_command( + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/block + COMMAND ${TURN_OFF_LSAN} + COMMAND ${GENERATE_TLB_CMD} -o block-auto -n block::gen -z block.tlb + COMMENT "Generate block tlb source files" + OUTPUT ${TLB_BLOCK_AUTO} + DEPENDS tlbc block/block.tlb + ) + else() + add_custom_command( + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/block + COMMAND ${TURN_OFF_LSAN} + COMMENT "Generate block tlb source files" + OUTPUT ${TLB_BLOCK_AUTO} + DEPENDS tlbc block/block.tlb + ) + endif() add_custom_target(tlb_generate_block DEPENDS ${TLB_BLOCK_AUTO}) add_dependencies(ton_block tlb_generate_block) diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index 1c490383..3e2ff81a 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -982,7 +982,7 @@ struct ShardIdent::Record { int shard_pfx_bits; int workchain_id; unsigned long long shard_prefix; - Record() : shard_pfx_bits(-1) { + Record() : shard_pfx_bits(-1), workchain_id(ton::workchainInvalid), shard_prefix(0) { } Record(int _pfxlen, int _wcid, unsigned long long _pfx) : shard_pfx_bits(_pfxlen), workchain_id(_wcid), shard_prefix(_pfx) { diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index f1a1c948..c1bd9c66 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -819,7 +819,7 @@ vmc_pushint$1111 value:int32 next:^VmCont = VmCont; // // DNS RECORDS // -_ (HashmapE 256 DNSRecord) = DNS_RecordSet; +_ (HashmapE 256 ^DNSRecord) = DNS_RecordSet; chunk_ref$_ {n:#} ref:^(TextChunks (n + 1)) = TextChunkRef (n + 1); chunk_ref_empty$_ = TextChunkRef 0; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index f5fb18f1..3b2334d6 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -327,8 +327,7 @@ bool Account::unpack(Ref shard_account, Ref extra, block::gen::t_ShardAccount.print(std::cerr, *shard_account); } block::gen::ShardAccount::Record acc_info; - if (!(block::gen::t_ShardAccount.validate_csr(shard_account) && - block::tlb::t_ShardAccount.validate_csr(shard_account) && tlb::unpack_exact(shard_account.write(), acc_info))) { + if (!(block::tlb::t_ShardAccount.validate_csr(shard_account) && tlb::unpack_exact(shard_account.write(), acc_info))) { LOG(ERROR) << "account " << addr.to_hex() << " state is invalid"; return false; } @@ -2013,7 +2012,6 @@ bool Transaction::compute_state() { std::cerr << "new account state: "; block::gen::t_Account.print_ref(std::cerr, new_total_state); } - CHECK(block::gen::t_Account.validate_ref(new_total_state)); CHECK(block::tlb::t_Account.validate_ref(new_total_state)); return true; } diff --git a/crypto/common/bigint.hpp b/crypto/common/bigint.hpp index 94da3a13..5fa8e2da 100644 --- a/crypto/common/bigint.hpp +++ b/crypto/common/bigint.hpp @@ -176,6 +176,7 @@ class AnyIntView { public: enum { word_bits = Tr::word_bits, word_shift = Tr::word_shift }; typedef typename Tr::word_t word_t; + typedef typename Tr::uword_t uword_t; int& n_; PropagateConstSpan digits; @@ -320,7 +321,7 @@ class BigIntG { digits[0] = x; normalize_bool(); } else { - digits[0] = ((x + Tr::Half) & (Tr::Base - 1)) - Tr::Half; + digits[0] = ((x ^ Tr::Half) & (Tr::Base - 1)) - Tr::Half; digits[n++] = (x >> Tr::word_shift) + (digits[0] < 0); } } @@ -675,7 +676,7 @@ class BigIntG { return n > 0 && !(digits[0] & 1); } word_t mod_pow2_short(int pow) const { - return n > 0 ? digits[0] & ((1 << pow) - 1) : 0; + return n > 0 ? digits[0] & ((1ULL << pow) - 1) : 0; } private: @@ -764,7 +765,7 @@ bool AnyIntView::add_pow2_any(int exponent, int factor) { while (size() <= k) { digits[inc_size()] = 0; } - digits[k] += ((word_t)factor << dm.rem); + digits[k] += factor * ((word_t)1 << dm.rem); return true; } @@ -969,7 +970,7 @@ bool AnyIntView::add_mul_any(const AnyIntView& yp, const AnyIntView& if (hi && hi != -1) { return invalidate_bool(); } - digits[size() - 1] += (hi << word_shift); + digits[size() - 1] += ((uword_t)hi << word_shift); } return true; } @@ -1014,7 +1015,7 @@ int AnyIntView::sgn_un_any() const { } int i = size() - 2; do { - v <<= word_shift; + v *= Tr::Base; word_t w = digits[i]; if (w >= -v + Tr::MaxDenorm) { return 1; @@ -1059,7 +1060,7 @@ typename Tr::word_t AnyIntView::to_long_any() const { } else if (size() == 1) { return digits[0]; } else { - word_t v = digits[0] + (digits[1] << word_shift); // approximation mod 2^64 + word_t v = (uword_t)digits[0] + ((uword_t)digits[1] << word_shift); // approximation mod 2^64 word_t w = (v & (Tr::Base - 1)) - digits[0]; w >>= word_shift; w += (v >> word_shift); // excess of approximation divided by Tr::Base @@ -1120,7 +1121,7 @@ int AnyIntView::cmp_un_any(const AnyIntView& yp) const { return -1; } while (xn > yn) { - v <<= word_shift; + v *= Tr::Base; word_t w = T::eval(digits[--xn]); if (w >= -v + Tr::MaxDenorm) { return 1; @@ -1137,7 +1138,7 @@ int AnyIntView::cmp_un_any(const AnyIntView& yp) const { return -1; } while (yn > xn) { - v <<= word_shift; + v *= Tr::Base; word_t w = yp.digits[--yn]; if (w <= v - Tr::MaxDenorm) { return 1; @@ -1150,7 +1151,7 @@ int AnyIntView::cmp_un_any(const AnyIntView& yp) const { v = 0; } while (--xn >= 0) { - v <<= word_shift; + v *= Tr::Base; word_t w = T::eval(digits[xn]) - yp.digits[xn]; if (w >= -v + Tr::MaxDenorm) { return 1; @@ -1197,7 +1198,7 @@ int AnyIntView::divmod_tiny_any(int y) { } int rem = 0; for (int i = size() - 1; i >= 0; i--) { - auto divmod = std::div(digits[i] + ((word_t)rem << word_shift), (word_t)y); + auto divmod = std::div(digits[i] + ((uword_t)rem << word_shift), (word_t)y); digits[i] = divmod.quot; rem = (int)divmod.rem; if ((rem ^ y) < 0 && rem) { @@ -1267,7 +1268,7 @@ bool AnyIntView::mul_add_short_any(word_t y, word_t z) { z += (digits[size() - 1] >> word_shift); digits[size() - 1] &= Tr::Base - 1; if (!z || z == -1) { - digits[size() - 1] += (z << word_shift); + digits[size() - 1] += ((uword_t)z << word_shift); return true; } else { return false; @@ -1338,7 +1339,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, while (--i >= 0) { Tr::sub_mul(&digits[k + i + 1], &digits[k + i], q, yp.digits[i]); } - digits[size() - 1] += (hi << word_shift); + digits[size() - 1] += ((uword_t)hi << word_shift); } } else { quot.set_size(1); @@ -1351,7 +1352,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, Tr::sub_mul(&digits[k + i + 1], &digits[k + i], q, yp.digits[i]); } dec_size(); - digits[size() - 1] += (digits[size()] << word_shift); + digits[size() - 1] += ((uword_t)digits[size()] << word_shift); } if (size() >= yp.size() - 1) { assert(size() <= yp.size()); @@ -1455,7 +1456,7 @@ bool AnyIntView::mod_pow2_any(int exponent) { dec_size(); q += word_shift; } - word_t pow = ((word_t)1 << q); + uword_t pow = ((uword_t)1 << q); word_t v = digits[size() - 1] & (pow - 1); if (!v) { int k = size() - 1; @@ -1485,7 +1486,7 @@ bool AnyIntView::mod_pow2_any(int exponent) { return true; } else if (v >= Tr::Half && size() < max_size()) { word_t w = (((v >> (word_shift - 1)) + 1) >> 1); - digits[size() - 1] = v - (w << word_shift); + digits[size() - 1] = (uword_t)v - ((uword_t)w << word_shift); digits[inc_size()] = w; return true; } else { @@ -1623,7 +1624,7 @@ bool AnyIntView::lshift_any(int exponent) { } else if (v != -1) { return invalidate_bool(); } else { - digits[size() - 1] += (v << word_shift); + digits[size() - 1] += ((uword_t)v << word_shift); } } if (q) { @@ -1750,7 +1751,7 @@ int AnyIntView::bit_size_any(bool sgnd) const { int k = size() - 1; word_t q = digits[k]; if (k > 0 && q < Tr::MaxDenorm / 2) { - q <<= word_shift; + q *= Tr::Base; q += digits[--k]; } if (!k) { @@ -1766,7 +1767,7 @@ int AnyIntView::bit_size_any(bool sgnd) const { } else if (q <= -Tr::MaxDenorm / 2) { return s; } - q <<= word_shift; + q *= Tr::Base; q += digits[--k]; } return q >= 0 ? s + 1 : s; @@ -1774,7 +1775,7 @@ int AnyIntView::bit_size_any(bool sgnd) const { int k = size() - 1; word_t q = digits[k]; if (k > 0 && q > -Tr::MaxDenorm / 2) { - q <<= word_shift; + q *= Tr::Base; q += digits[--k]; } if (!k) { @@ -1790,7 +1791,7 @@ int AnyIntView::bit_size_any(bool sgnd) const { } else if (q <= -Tr::MaxDenorm / 2) { return s + 1; } - q <<= word_shift; + q *= Tr::Base; q += digits[--k]; } return q >= 0 ? s : s + 1; @@ -1817,7 +1818,7 @@ bool AnyIntView::export_bytes_any(unsigned char* buff, std::size_t buff_size for (int i = 0; i < size(); i++) { if ((word_shift & 7) && word_shift + 8 >= word_bits && k >= word_bits - word_shift - 1) { int k1 = 8 - k; - v += (digits[i] << k) & 0xff; + v += ((uword_t)digits[i] << k) & 0xff; if (ptr > buff) { *--ptr = (unsigned char)(v & 0xff); } else if ((unsigned char)(v & 0xff) != s) { @@ -1827,7 +1828,7 @@ bool AnyIntView::export_bytes_any(unsigned char* buff, std::size_t buff_size v += (digits[i] >> k1); k += word_shift - 8; } else { - v += (digits[i] << k); + v += ((uword_t)digits[i] << k); k += word_shift; } while (k >= 8) { @@ -1868,7 +1869,7 @@ bool AnyIntView::export_bytes_lsb_any(unsigned char* buff, std::size_t buff_ for (int i = 0; i < size(); i++) { if ((word_shift & 7) && word_shift + 8 >= word_bits && k >= word_bits - word_shift - 1) { int k1 = 8 - k; - v += (digits[i] << k) & 0xff; + v += ((uword_t)digits[i] << k) & 0xff; if (buff < end) { *buff++ = (unsigned char)(v & 0xff); } else if ((unsigned char)(v & 0xff) != s) { @@ -1878,7 +1879,7 @@ bool AnyIntView::export_bytes_lsb_any(unsigned char* buff, std::size_t buff_ v += (digits[i] >> k1); k += word_shift - 8; } else { - v += (digits[i] << k); + v += ((uword_t)digits[i] << k); k += word_shift; } while (k >= 8) { @@ -1922,7 +1923,7 @@ bool AnyIntView::export_bits_any(unsigned char* buff, int offs, unsigned bit return false; } } - td::bitstring::bits_store_long_top(buff, offs, v << (64 - bits), bits); + td::bitstring::bits_store_long_top(buff, offs, (unsigned long long)v << (64 - bits), bits); } else { if (!sgnd && v < 0) { return false; @@ -1945,7 +1946,7 @@ bool AnyIntView::export_bits_any(unsigned char* buff, int offs, unsigned bit for (int i = 0; i < size(); i++) { if (word_shift + 8 >= word_bits && k >= word_bits - word_shift - 1) { int k1 = 8 - k; - v += (digits[i] << k) & 0xff; + v += ((uword_t)digits[i] << k) & 0xff; if (ptr > buff) { if (--ptr > buff) { *ptr = (unsigned char)(v & 0xff); @@ -1963,7 +1964,7 @@ bool AnyIntView::export_bits_any(unsigned char* buff, int offs, unsigned bit v += (digits[i] >> k1); k += word_shift - 8; } else { - v += (digits[i] << k); + v += ((uword_t)digits[i] << k); k += word_shift; } while (k >= 8) { @@ -2028,7 +2029,7 @@ bool AnyIntView::import_bytes_any(const unsigned char* buff, std::size_t buf return invalidate_bool(); } } - v |= (((word_t) * --ptr) << k); + v |= (((uword_t) * --ptr) << k); k += 8; } if (s) { @@ -2043,7 +2044,9 @@ bool AnyIntView::import_bits_any(const unsigned char* buff, int offs, unsign if (bits < word_shift) { set_size(1); unsigned long long val = td::bitstring::bits_load_long_top(buff, offs, bits); - if (sgnd) { + if (bits == 0) { + digits[0] = 0; + } else if (sgnd) { digits[0] = ((long long)val >> (64 - bits)); } else { digits[0] = (val >> (64 - bits)); diff --git a/crypto/common/bitstring.cpp b/crypto/common/bitstring.cpp index 0a273949..aabc6984 100644 --- a/crypto/common/bitstring.cpp +++ b/crypto/common/bitstring.cpp @@ -191,7 +191,7 @@ void bits_memcpy(unsigned char* to, int to_offs, const unsigned char* from, int *to++ = (unsigned char)(acc >> b); } if (b > 0) { - *to = (unsigned char)((*to & (0xff >> b)) | ((int)acc << (8 - b))); + *to = (unsigned char)((*to & (0xff >> b)) | ((unsigned)acc << (8 - b))); } } } @@ -301,7 +301,7 @@ std::size_t bits_memscan(const unsigned char* ptr, int offs, std::size_t bit_cou ptr++; } while (rem >= 8 && !td::is_aligned_pointer<8>(ptr)) { - v = ((*ptr++ ^ xor_val) << 24); + v = ((unsigned)(*ptr++ ^ xor_val) << 24); // std::cerr << "[B] rem=" << rem << " ptr=" << (const void*)(ptr - 1) << " v=" << std::hex << v << std::dec << std::endl; if (v) { return bit_count - rem + td::count_leading_zeroes_non_zero32(v); @@ -319,7 +319,7 @@ std::size_t bits_memscan(const unsigned char* ptr, int offs, std::size_t bit_cou rem -= 64; } while (rem >= 8) { - v = ((*ptr++ ^ xor_val) << 24); + v = ((unsigned)(*ptr++ ^ xor_val) << 24); // std::cerr << "[D] rem=" << rem << " ptr=" << (const void*)(ptr - 1) << " v=" << std::hex << v << std::dec << std::endl; if (v) { return bit_count - rem + td::count_leading_zeroes_non_zero32(v); @@ -327,7 +327,7 @@ std::size_t bits_memscan(const unsigned char* ptr, int offs, std::size_t bit_cou rem -= 8; } if (rem > 0) { - v = ((*ptr ^ xor_val) << 24); + v = ((unsigned)(*ptr ^ xor_val) << 24); // std::cerr << "[E] rem=" << rem << " ptr=" << (const void*)ptr << " v=" << std::hex << v << std::dec << std::endl; c = td::count_leading_zeroes32(v); return c < rem ? bit_count - rem + c : bit_count; @@ -505,7 +505,7 @@ unsigned long long bits_load_long_top(ConstBitPtr from, unsigned top_bits) { } unsigned long long bits_load_ulong(ConstBitPtr from, unsigned bits) { - return bits_load_long_top(from, bits) >> (64 - bits); + return bits == 0 ? 0 : bits_load_long_top(from, bits) >> (64 - bits); } long long bits_load_long(ConstBitPtr from, unsigned bits) { diff --git a/crypto/func/auto-tests/run_tests.py b/crypto/func/auto-tests/run_tests.py new file mode 100644 index 00000000..ae9c990c --- /dev/null +++ b/crypto/func/auto-tests/run_tests.py @@ -0,0 +1,91 @@ +import os +import os.path +import subprocess +import sys +import tempfile + +def getenv(name, default=None): + if name in os.environ: + return os.environ[name] + if default is None: + print("Environment variable", name, "is not set", file=sys.stderr) + exit(1) + return default + +FUNC_EXECUTABLE = getenv("FUNC_EXECUTABLE", "func") +FIFT_EXECUTABLE = getenv("FIFT_EXECUTABLE", "fift") +#FUNC_STDLIB = getenv("FUNC_STDLIB") +FIFT_LIBS = getenv("FIFT_LIBS") +TMP_DIR = tempfile.mkdtemp() +COMPILED_FIF = os.path.join(TMP_DIR, "compiled.fif") +RUNNER_FIF = os.path.join(TMP_DIR, "runner.fif") + +if len(sys.argv) != 2: + print("Usage : run_tests.py tests_dir", file=sys.stderr) + exit(1) +TESTS_DIR = sys.argv[1] + +class ExecutionError(Exception): + pass + +def compile_func(f): + res = subprocess.run([FUNC_EXECUTABLE, "-o", COMPILED_FIF, "-SPA", f], capture_output=True, timeout=10) + if res.returncode != 0: + raise ExecutionError(str(res.stderr, "utf-8")) + +def run_runner(): + res = subprocess.run([FIFT_EXECUTABLE, "-I", FIFT_LIBS, RUNNER_FIF], capture_output=True, timeout=10) + if res.returncode != 0: + raise ExecutionError(str(res.stderr, "utf-8")) + s = str(res.stdout, "utf-8") + s = [x.strip() for x in s.split("\n")] + return [x for x in s if x != ""] + +tests = [s for s in os.listdir(TESTS_DIR) if s.endswith(".fc")] +tests.sort() +print("Found", len(tests), "tests", file=sys.stderr) +for ti, tf in enumerate(tests): + print("Running test %d/%d: %s" % (ti + 1, len(tests), tf), file=sys.stderr) + tf = os.path.join(TESTS_DIR, tf) + try: + compile_func(tf) + except ExecutionError as e: + print(file=sys.stderr) + print("Compilation error", file=sys.stderr) + print(e, file=sys.stderr) + exit(2) + with open(tf, "r") as fd: + lines = fd.readlines() + cases = [] + for s in lines: + s = [x.strip() for x in s.split("|")] + if len(s) == 4 and s[0].strip() == "TESTCASE": + cases.append(s[1:]) + if len(cases) == 0: + print(file=sys.stderr) + print("Error: no test cases", file=sys.stderr) + exit(2) + + with open(RUNNER_FIF, "w") as f: + print("\"%s\" include = %d);" % (var, self.n), file=f) + +def write_function(f, name, body, inline=False, inline_ref=False, method_id=None): + print("_ %s(int x)" % name, file=f, end="") + if inline: + print(" inline", file=f, end="") + if inline_ref: + print(" inline_ref", file=f, end="") + if method_id is not None: + print(" method_id(%d)" % method_id, file=f, end="") + print(" {", file=f) + for i in range(VAR_CNT): + print(" int v%d = 0;" % i, file=f) + body.write(f, 1); + print("}", file=f) + +def gen_code(xl, xr, with_return, loop_depth=0): + code = [] + for _ in range(random.randint(0, 2)): + if random.randint(0, 3) == 0 and loop_depth < 3: + c = gen_code(xl, xr, False, loop_depth + 1) + code.append(CodeRepeat(random.randint(0, 3), c, random.randint(0, 2))) + elif xr - xl > 1: + xmid = random.randrange(xl + 1, xr) + ret = random.choice((0, 0, 0, 0, 0, 1, 2)) + c1 = gen_code(xl, xmid, ret == 1, loop_depth) + if random.randrange(5) == 0: + c2 = CodeEmpty() + else: + c2 = gen_code(xmid, xr, ret == 2, loop_depth) + code.append(CodeIfRange(xl, xmid, c1, c2)) + if with_return: + if xr - xl == 1: + code.append(CodeReturn(random.randrange(10**9))) + else: + xmid = random.randrange(xl + 1, xr) + c1 = gen_code(xl, xmid, True, loop_depth) + c2 = gen_code(xmid, xr, True, loop_depth) + code.append(CodeIfRange(xl, xmid, c1, c2)) + for _ in range(random.randint(0, 3)): + pos = random.randint(0, len(code)) + code.insert(pos, CodeAdd(random.randrange(VAR_CNT), random.randint(0, 10**6))) + if len(code) == 0: + return CodeEmpty() + return CodeBlock(code) + +class ExecutionError(Exception): + pass + +def compile_func(fc, fif): + res = subprocess.run([FUNC_EXECUTABLE, "-o", fif, "-SPA", fc], capture_output=True) + if res.returncode != 0: + raise ExecutionError(str(res.stderr, "utf-8")) + +def runvm(compiled_fif, xl, xr): + runner = os.path.join(TMP_DIR, "runner.fif") + with open(runner, "w") as f: + print("\"%s\" include a) { + x -= 1; + z = 1; + } + return (y, z); +} + +{- + method_id | in | out +TESTCASE | 0 | 101 15 | 100 1 +TESTCASE | 0 | 101 14 | 100 1 +TESTCASE | 0 | 101 10 | 100 0 +TESTCASE | 0 | 100 10 | 100 0 +-} diff --git a/crypto/func/auto-tests/tests/a6.fc b/crypto/func/auto-tests/tests/a6.fc new file mode 100644 index 00000000..05a49fab --- /dev/null +++ b/crypto/func/auto-tests/tests/a6.fc @@ -0,0 +1,89 @@ +(int, int) f(int a, int b, int c, int d, int e, int f) { + ;; solve a 2x2 linear equation + int D = a * d - b * c; + int Dx = e * d - b * f; + int Dy = a * f - e * c; + return (Dx / D, Dy / D); +} + +int calc_phi() { + var n = 1; + repeat (70) { n *= 10; } + var p = var q = 1; + do { + (p, q) = (q, p + q); + } until (q > n); + return muldivr(p, n, q); +} + +int calc_sqrt2() { + var n = 1; + repeat (70) { n *= 10; } + var p = var q = 1; + do { + var t = p + q; + (p, q) = (q, t + q); + } until (q > n); + return muldivr(p, n, q); +} + +var calc_root(m) { + int base = 1; + repeat(70) { base *= 10; } + var (a, b, c) = (1, 0, - m); + var (p1, q1, p2, q2) = (1, 0, 0, 1); + do { + int k = -1; + var (a1, b1, c1) = (0, 0, 0); + do { + k += 1; + (a1, b1, c1) = (a, b, c); + c += b; + c += b += a; + } until (c > 0); + (a, b, c) = (- c1, - b1, - a1); + (p1, q1) = (k * p1 + q1, p1); + (p2, q2) = (k * p2 + q2, p2); + } until (p1 > base); + return (p1, q1, p2, q2); +} + +{- +operator _/%_ infix 20; + +(int, int) ((int x) /% (int y)) { + return (x / y, x % y); +} + +(int, int) _/%_ (int x, int y) { + return (x / y, x % y); +} +-} + +int ataninv(int base, int q) { ;; computes base*atan(1/q) + base ~/= q; + q *= - q; + int sum = 0; + int n = 1; + do { + sum += base ~/ n; + base ~/= q; + n += 2; + } until base == 0; + return sum; +} + +int calc_pi() { + int base = 64; + repeat (70) { base *= 10; } + return (ataninv(base << 2, 5) - ataninv(base, 239)) ~>> 4; +} + +int main() { + return calc_pi(); +} + +{- + method_id | in | out +TESTCASE | 0 | | 31415926535897932384626433832795028841971693993751058209749445923078164 +-} diff --git a/crypto/func/auto-tests/tests/a6_1.fc b/crypto/func/auto-tests/tests/a6_1.fc new file mode 100644 index 00000000..b6341df0 --- /dev/null +++ b/crypto/func/auto-tests/tests/a6_1.fc @@ -0,0 +1,16 @@ +(int, int) main(int a, int b, int c, int d, int e, int f) { + int D = a * d - b * c; + int Dx = e * d - b * f; + int Dy = a * f - e * c; + return (Dx / D, Dy / D); +} + +{- + method_id | in | out +TESTCASE | 0 | 1 1 1 -1 10 6 | 8 2 +TESTCASE | 0 | 817 -31 624 -241 132272 272276 | 132 -788 +TESTCASE | 0 | -886 562 498 -212 -36452 -68958 | -505 -861 +TESTCASE | 0 | 448 -433 -444 792 150012 -356232 | -218 -572 +TESTCASE | 0 | -40 -821 433 -734 -721629 -741724 | -206 889 +TESTCASE | 0 | -261 -98 -494 868 -166153 733738 | 263 995 +-} diff --git a/crypto/func/auto-tests/tests/a6_5.fc b/crypto/func/auto-tests/tests/a6_5.fc new file mode 100644 index 00000000..06b5cc9d --- /dev/null +++ b/crypto/func/auto-tests/tests/a6_5.fc @@ -0,0 +1,24 @@ +var twice(f, x) { + return f (f x); +} + +_ sqr(x) { + return x * x; +} + +var main(x) { + var f = sqr; + return twice(f, x) * f(x); +} + +var pow6(x) method_id(4) { + return twice(sqr, x) * sqr(x); +} + +{- + method_id | in | out +TESTCASE | 0 | 3 | 729 +TESTCASE | 0 | 10 | 1000000 +TESTCASE | 4 | 3 | 729 +TESTCASE | 4 | 10 | 1000000 +-} diff --git a/crypto/func/auto-tests/tests/a7.fc b/crypto/func/auto-tests/tests/a7.fc new file mode 100644 index 00000000..356759d4 --- /dev/null +++ b/crypto/func/auto-tests/tests/a7.fc @@ -0,0 +1,24 @@ +() main() { } +int steps(int x) method_id(1) { + var n = 0; + while (x > 1) { + n += 1; + if (x & 1) { + x = 3 * x + 1; + } else { + x >>= 1; + } + } + return n; +} + +{- + method_id | in | out +TESTCASE | 1 | 1 | 0 +TESTCASE | 1 | 2 | 1 +TESTCASE | 1 | 5 | 5 +TESTCASE | 1 | 19 | 20 +TESTCASE | 1 | 27 | 111 +TESTCASE | 1 | 100 | 25 +-} + diff --git a/crypto/func/auto-tests/tests/c2.fc b/crypto/func/auto-tests/tests/c2.fc new file mode 100644 index 00000000..27f8b88f --- /dev/null +++ b/crypto/func/auto-tests/tests/c2.fc @@ -0,0 +1,17 @@ +global ((int, int) -> int) op; + +int check_assoc(int a, int b, int c) { + return op(op(a, b), c) == op(a, op(b, c)); +} + +int main(int x, int y, int z) { + op = _+_; + return check_assoc(x, y, z); +} + +{- + method_id | in | out +TESTCASE | 0 | 2 3 9 | -1 +TESTCASE | 0 | 11 22 44 | -1 +TESTCASE | 0 | -1 -10 -20 | -1 +-} diff --git a/crypto/func/auto-tests/tests/c2_1.fc b/crypto/func/auto-tests/tests/c2_1.fc new file mode 100644 index 00000000..ef7183b6 --- /dev/null +++ b/crypto/func/auto-tests/tests/c2_1.fc @@ -0,0 +1,14 @@ +_ check_assoc(op, a, b, c) { + return op(op(a, b), c) == op(a, op(b, c)); +} + +int main(int x, int y, int z) { + return check_assoc(_+_, x, y, z); +} + +{- + method_id | in | out +TESTCASE | 0 | 2 3 9 | -1 +TESTCASE | 0 | 11 22 44 | -1 +TESTCASE | 0 | -1 -10 -20 | -1 +-} diff --git a/crypto/func/auto-tests/tests/co1.fc b/crypto/func/auto-tests/tests/co1.fc new file mode 100644 index 00000000..44067243 --- /dev/null +++ b/crypto/func/auto-tests/tests/co1.fc @@ -0,0 +1,60 @@ +const int1 = 1, int2 = 2; + +const int int101 = 101; +const int int111 = 111; + +const int1r = int1; + +const str1 = "const1", str2 = "aabbcc"s; + +const slice str2r = str2; + +const str1int = 0x636f6e737431; +const str2int = 0xAABBCC; + +const int nibbles = 4; + +int iget1() { return int1; } +int iget2() { return int2; } +int iget3() { return int1 + int2; } + +int iget1r() { return int1r; } + +slice sget1() { return str1; } +slice sget2() { return str2; } +slice sget2r() { return str2r; } + +const int int240 = ((int1 + int2) * 10) << 3; + +int iget240() { return int240; } + +builder newc() asm "NEWC"; +slice endcs(builder b) asm "ENDC" "CTOS"; +int sdeq (slice s1, slice s2) asm "SDEQ"; +builder stslicer(builder b, slice s) asm "STSLICER"; + +_ main() { + int i1 = iget1(); + int i2 = iget2(); + int i3 = iget3(); + + throw_unless(int101, i1 == 1); + throw_unless(102, i2 == 2); + throw_unless(103, i3 == 3); + + slice s1 = sget1(); + slice s2 = sget2(); + slice s3 = newc().stslicer(str1).stslicer(str2r).endcs(); + + throw_unless(int111, sdeq(s1, newc().store_uint(str1int, 12 * nibbles).endcs())); + throw_unless(112, sdeq(s2, newc().store_uint(str2int, 6 * nibbles).endcs())); + throw_unless(113, sdeq(s3, newc().store_uint(0x636f6e737431AABBCC, 18 * nibbles).endcs())); + + int i4 = iget240(); + throw_unless(104, i4 == 240); + return 0; +} + +{- +TESTCASE | 0 | | 0 +-} diff --git a/crypto/func/auto-tests/tests/code_after_ifelse.fc b/crypto/func/auto-tests/tests/code_after_ifelse.fc new file mode 100644 index 00000000..49e082c7 --- /dev/null +++ b/crypto/func/auto-tests/tests/code_after_ifelse.fc @@ -0,0 +1,19 @@ +int foo(int x) inline method_id(1) { + if (x == 1) { + return 111; + } else { + x *= 2; + } + return x + 1; +} +(int, int) main(int x) { + return (foo(x), 222); +} + +{- + method_id | in | out +TESTCASE | 1 | 1 | 111 +TESTCASE | 1 | 3 | 7 +TESTCASE | 0 | 1 | 111 222 +TESTCASE | 0 | 3 | 7 222 +-} diff --git a/crypto/func/auto-tests/tests/inline_big.fc b/crypto/func/auto-tests/tests/inline_big.fc new file mode 100644 index 00000000..61ad436e --- /dev/null +++ b/crypto/func/auto-tests/tests/inline_big.fc @@ -0,0 +1,61 @@ +int foo(int x) inline { + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + return x; +} + +(int) main(int x) { + return foo(x) * 10 + 5; +} +{- + method_id | in | out +TESTCASE | 0 | 9 | 9111111111111111111111111111111111111111111111111115 +-} diff --git a/crypto/func/auto-tests/tests/inline_if.fc b/crypto/func/auto-tests/tests/inline_if.fc new file mode 100644 index 00000000..390f0fd3 --- /dev/null +++ b/crypto/func/auto-tests/tests/inline_if.fc @@ -0,0 +1,26 @@ +int foo1(int x) { + if (x == 1) { + return 1; + } + return 2; +} +int foo2(int x) inline { + if (x == 1) { + return 11; + } + return 22; +} +int foo3(int x) inline_ref { + if (x == 1) { + return 111; + } + return 222; +} +(int, int, int) main(int x) { + return (foo1(x) + 1, foo2(x) + 1, foo3(x) + 1); +} +{- + method_id | in | out +TESTCASE | 0 | 1 | 2 12 112 +TESTCASE | 0 | 2 | 3 23 223 +-} diff --git a/crypto/func/auto-tests/tests/inline_loops.fc b/crypto/func/auto-tests/tests/inline_loops.fc new file mode 100644 index 00000000..9f1f45fc --- /dev/null +++ b/crypto/func/auto-tests/tests/inline_loops.fc @@ -0,0 +1,43 @@ +global int g; + +_ foo_repeat() impure inline { + g = 1; + repeat(5) { + g *= 2; + } +} + +int foo_until() impure inline { + g = 1; + int i = 0; + do { + g *= 2; + i += 1; + } until (i >= 8); + return i; +} + +int foo_while() impure inline { + g = 1; + int i = 0; + while (i < 10) { + g *= 2; + i += 1; + } + return i; +} + +_ main() { + foo_repeat(); + int x = g; + foo_until(); + int y = g; + foo_while(); + int z = g; + return (x, y, z); +} + +{- + method_id | in | out +TESTCASE | 0 | | 32 256 1024 +-} diff --git a/crypto/func/auto-tests/tests/method_id.fc b/crypto/func/auto-tests/tests/method_id.fc new file mode 100644 index 00000000..b4e0cd3b --- /dev/null +++ b/crypto/func/auto-tests/tests/method_id.fc @@ -0,0 +1,12 @@ +int foo1() method_id(1) { return 111; } +int foo2() method_id(3) { return 222; } +int foo3() method_id(10) { return 333; } +int main() { return 999; } + +{- + method_id | in | out +TESTCASE | 1 | | 111 +TESTCASE | 3 | | 222 +TESTCASE | 10 | | 333 +TESTCASE | 0 | | 999 +-} diff --git a/crypto/func/auto-tests/tests/s1.fc b/crypto/func/auto-tests/tests/s1.fc new file mode 100644 index 00000000..1541943d --- /dev/null +++ b/crypto/func/auto-tests/tests/s1.fc @@ -0,0 +1,54 @@ +slice ascii_slice() method_id { + return "string"; +} + +slice raw_slice() method_id { + return "abcdef"s; +} + +slice addr_slice() method_id { + return "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a; +} + +int string_hex() method_id { + return "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"u; +} + +int string_minihash() method_id { + return "transfer(slice, int)"h; +} + +int string_maxihash() method_id { + return "transfer(slice, int)"H; +} + +int string_crc32() method_id { + return "transfer(slice, int)"c; +} + +builder newc() asm "NEWC"; +slice endcs(builder b) asm "ENDC" "CTOS"; +int sdeq (slice s1, slice s2) asm "SDEQ"; + +_ main() { + slice s_ascii = ascii_slice(); + slice s_raw = raw_slice(); + slice s_addr = addr_slice(); + int i_hex = string_hex(); + int i_mini = string_minihash(); + int i_maxi = string_maxihash(); + int i_crc = string_crc32(); + throw_unless(101, sdeq(s_ascii, newc().store_uint(0x737472696E67, 12 * 4).endcs())); + throw_unless(102, sdeq(s_raw, newc().store_uint(0xABCDEF, 6 * 4).endcs())); + throw_unless(103, sdeq(s_addr, newc().store_uint(4, 3).store_int(-1, 8) + .store_uint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs())); + throw_unless(104, i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435); + throw_unless(105, i_mini == 0x7a62e8a8); + throw_unless(106, i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979); + throw_unless(107, i_crc == 2235694568); + return 0; +} + +{- +TESTCASE | 0 | | 0 +-} diff --git a/crypto/func/auto-tests/tests/unbalanced_ret.fc b/crypto/func/auto-tests/tests/unbalanced_ret.fc new file mode 100644 index 00000000..0e4ef6ae --- /dev/null +++ b/crypto/func/auto-tests/tests/unbalanced_ret.fc @@ -0,0 +1,17 @@ +(int, int) main(int x) { + int y = 5; + if (x < 0) { + x *= 2; + y += 1; + if (x == -10) { + return (111, 0); + } + } + return (x + 1, y); +} +{- + method_id | in | out +TESTCASE | 0 | 10 | 11 5 +TESTCASE | 0 | -5 | 111 0 +TESTCASE | 0 | -4 | -7 6 +-} diff --git a/crypto/func/auto-tests/tests/unbalanced_ret_inline.fc b/crypto/func/auto-tests/tests/unbalanced_ret_inline.fc new file mode 100644 index 00000000..6d169345 --- /dev/null +++ b/crypto/func/auto-tests/tests/unbalanced_ret_inline.fc @@ -0,0 +1,18 @@ +int foo(int x) inline { + if (x < 0) { + x *= 2; + if (x == -10) { + return 111; + } + } + return x + 1; +} +int main(int x) { + return foo(x) * 10; +} +{- + method_id | in | out +TESTCASE | 0 | 10 | 110 +TESTCASE | 0 | -5 | 1110 +TESTCASE | 0 | -4 | -70 +-} diff --git a/crypto/func/auto-tests/tests/unbalanced_ret_loops.fc b/crypto/func/auto-tests/tests/unbalanced_ret_loops.fc new file mode 100644 index 00000000..104ec00d --- /dev/null +++ b/crypto/func/auto-tests/tests/unbalanced_ret_loops.fc @@ -0,0 +1,48 @@ +_ main() { } + +int foo_repeat(int x) method_id(1) { + repeat(10) { + x += 10; + if (x >= 100) { + return x; + } + } + return -1; +} + +int foo_while(int x) method_id(2) { + int i = 0; + while (i < 10) { + x += 10; + if (x >= 100) { + return x; + } + i += 1; + } + return -1; +} + +int foo_until(int x) method_id(3) { + int i = 0; + do { + x += 10; + if (x >= 100) { + return x; + } + i += 1; + } until (i >= 10); + return -1; +} + +{- + method_id | in | out +TESTCASE | 1 | 40 | 100 +TESTCASE | 1 | 33 | 103 +TESTCASE | 1 | -5 | -1 +TESTCASE | 2 | 40 | 100 +TESTCASE | 2 | 33 | 103 +TESTCASE | 2 | -5 | -1 +TESTCASE | 3 | 40 | 100 +TESTCASE | 3 | 33 | 103 +TESTCASE | 3 | -5 | -1 +-} diff --git a/crypto/func/auto-tests/tests/unbalanced_ret_nested.fc b/crypto/func/auto-tests/tests/unbalanced_ret_nested.fc new file mode 100644 index 00000000..7ab4bdf0 --- /dev/null +++ b/crypto/func/auto-tests/tests/unbalanced_ret_nested.fc @@ -0,0 +1,35 @@ +int foo(int y) { + if (y < 0) { + y *= 2; + if (y == -10) { + return 111; + } + } + return y + 1; +} +(int, int) bar(int x, int y) { + if (x < 0) { + y = foo(y); + x *= 2; + if (x == -10) { + return (111, y); + } + } + return (x + 1, y); +} +(int, int) main(int x, int y) { + (x, y) = bar(x, y); + return (x, y * 10); +} +{- + method_id | in | out +TESTCASE | 0 | 3 3 | 4 30 +TESTCASE | 0 | 3 -5 | 4 -50 +TESTCASE | 0 | 3 -4 | 4 -40 +TESTCASE | 0 | -5 3 | 111 40 +TESTCASE | 0 | -5 -5 | 111 1110 +TESTCASE | 0 | -5 -4 | 111 -70 +TESTCASE | 0 | -4 3 | -7 40 +TESTCASE | 0 | -4 -5 | -7 1110 +TESTCASE | 0 | -4 -4 | -7 -70 +-} diff --git a/crypto/func/auto-tests/tests/w1.fc b/crypto/func/auto-tests/tests/w1.fc new file mode 100644 index 00000000..3d6a8f69 --- /dev/null +++ b/crypto/func/auto-tests/tests/w1.fc @@ -0,0 +1,14 @@ +(int, int) main(int id) { + if (id > 0) { + if (id > 10) { + return (2 * id, 3 * id); + } + } + return (5, 6); +} +{- + method_id | in | out +TESTCASE | 0 | 0 | 5 6 +TESTCASE | 0 | 4 | 5 6 +TESTCASE | 0 | 11 | 22 33 +-} diff --git a/crypto/func/auto-tests/tests/w2.fc b/crypto/func/auto-tests/tests/w2.fc new file mode 100644 index 00000000..b9e5fb0e --- /dev/null +++ b/crypto/func/auto-tests/tests/w2.fc @@ -0,0 +1,18 @@ +_ f(cs) { + return (cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), + cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), + cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), + cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), + cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8)); +} + +_ main(cs) { + var (x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, + x11, x12, x13, x14, x15, x16, x17, x18, x19) = f(cs); + return x0 + x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19; +} +{- + method_id | in | out +TESTCASE | 0 | x{000102030405060708090a0b0c0d0e0f10111213} | 190 +-} diff --git a/crypto/func/auto-tests/tests/w6.fc b/crypto/func/auto-tests/tests/w6.fc new file mode 100644 index 00000000..9b91cdf7 --- /dev/null +++ b/crypto/func/auto-tests/tests/w6.fc @@ -0,0 +1,17 @@ +int main(int x) { + int i = 0; + ;; int f = false; + do { + i = i + 1; + if (i > 5) { + return 1; + } + int f = (i * i == 64); + } until (f); + return -1; +} + +{- + method_id | in | out +TESTCASE | 0 | 0 | 1 +-} diff --git a/crypto/func/auto-tests/tests/w7.fc b/crypto/func/auto-tests/tests/w7.fc new file mode 100644 index 00000000..991b3e64 --- /dev/null +++ b/crypto/func/auto-tests/tests/w7.fc @@ -0,0 +1,24 @@ +int test(int y) method_id(1) { + int x = 1; + if (y > 0) { + return 1; + } + return x > 0; +} + +int f(int y) method_id(2) { + if (y > 0) { + return 1; + } + return 2; +} + +_ main() { } + +{- + method_id | in | out +TESTCASE | 1 | 10 | 1 +TESTCASE | 1 | -5 | -1 +TESTCASE | 2 | 10 | 1 +TESTCASE | 2 | -5 | 2 +-} diff --git a/crypto/func/auto-tests/tests/w9.fc b/crypto/func/auto-tests/tests/w9.fc new file mode 100644 index 00000000..f299dec1 --- /dev/null +++ b/crypto/func/auto-tests/tests/w9.fc @@ -0,0 +1,14 @@ +_ main(s) { + var (z, t) = (17, s); + while (z > 0) { + t = s; + z -= 1; + } + return ~ t; +} + +{- + method_id | in | out +TESTCASE | 0 | 1 | -2 +TESTCASE | 0 | 5 | -6 +-} diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index 075d8f62..22f4282c 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -173,6 +173,74 @@ int emulate_mul(int a, int b) { return r; } +int emulate_and(int a, int b) { + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + if (any & VarDescr::_Zero) { + return VarDescr::ConstZero; + } + r |= both & (VarDescr::_Even | VarDescr::_Odd); + r |= both & (VarDescr::_Bit | VarDescr::_Bool); + if (both & VarDescr::_Odd) { + r |= VarDescr::_NonZero; + } + return r; +} + +int emulate_or(int a, int b) { + if (b & VarDescr::_Zero) { + return a; + } else if (a & VarDescr::_Zero) { + return b; + } + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + r |= any & VarDescr::_NonZero; + r |= any & VarDescr::_Odd; + r |= both & VarDescr::_Even; + return r; +} + +int emulate_xor(int a, int b) { + if (b & VarDescr::_Zero) { + return a; + } else if (a & VarDescr::_Zero) { + return b; + } + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + r |= both & VarDescr::_Even; + if (both & VarDescr::_Odd) { + r |= VarDescr::_Even; + } + return r; +} + +int emulate_not(int a) { + int f = VarDescr::_Even | VarDescr::_Odd; + if ((a & f) && (~a & f)) { + a ^= f; + } + f = VarDescr::_Pos | VarDescr::_Neg; + if ((a & f) && (~a & f)) { + a ^= f; + } + a &= ~(VarDescr::_Zero | VarDescr::_NonZero | VarDescr::_Bit); + return a; +} + int emulate_lshift(int a, int b) { if (((a | b) & VarDescr::_Nan) || !(~b & (VarDescr::_Neg | VarDescr::_NonZero))) { return VarDescr::_Int | VarDescr::_Nan; @@ -427,6 +495,57 @@ AsmOp compile_negate(std::vector& res, std::vector& args) { return exec_op("NEGATE", 1); } +AsmOp compile_and(std::vector& res, std::vector& args) { + assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const & y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_and(x.val, y.val); + return exec_op("AND", 2); +} + +AsmOp compile_or(std::vector& res, std::vector& args) { + assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const | y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_or(x.val, y.val); + return exec_op("OR", 2); +} + +AsmOp compile_xor(std::vector& res, std::vector& args) { + assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const ^ y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_xor(x.val, y.val); + return exec_op("XOR", 2); +} + +AsmOp compile_not(std::vector& res, std::vector& args) { + assert(res.size() == 1 && args.size() == 1); + VarDescr &r = res[0], &x = args[0]; + if (x.is_int_const()) { + r.set_const(~x.int_const); + x.unused(); + return push_const(r.int_const); + } + r.val = emulate_not(x.val); + return exec_op("NOT", 1); +} + AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y) { if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const * y.int_const); @@ -442,6 +561,7 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y) { if (y.always_zero() && x.always_finite()) { // dubious optimization: NaN * 0 = ? r.set_const(y.int_const); + x.unused(); return push_const(r.int_const); } if (*y.int_const == 1 && x.always_finite()) { @@ -468,6 +588,7 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y) { if (x.always_zero() && y.always_finite()) { // dubious optimization: NaN * 0 = ? r.set_const(x.int_const); + y.unused(); return push_const(r.int_const); } if (*x.int_const == 1 && y.always_finite()) { @@ -1000,10 +1121,10 @@ void define_builtins() { define_builtin_func("_>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, -1)); define_builtin_func("_~>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, 0)); define_builtin_func("_^>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, 1)); - define_builtin_func("_&_", arith_bin_op, AsmOp::Custom("AND", 2)); - define_builtin_func("_|_", arith_bin_op, AsmOp::Custom("OR", 2)); - define_builtin_func("_^_", arith_bin_op, AsmOp::Custom("XOR", 2)); - define_builtin_func("~_", arith_un_op, AsmOp::Custom("NOT", 1)); + define_builtin_func("_&_", arith_bin_op, compile_and); + define_builtin_func("_|_", arith_bin_op, compile_or); + define_builtin_func("_^_", arith_bin_op, compile_xor); + define_builtin_func("~_", arith_un_op, compile_not); define_builtin_func("^_+=_", arith_bin_op, compile_add); define_builtin_func("^_-=_", arith_bin_op, compile_sub); define_builtin_func("^_*=_", arith_bin_op, compile_mul); @@ -1017,9 +1138,9 @@ void define_builtins() { define_builtin_func("^_>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, -1)); define_builtin_func("^_~>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, 0)); define_builtin_func("^_^>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, 1)); - define_builtin_func("^_&=_", arith_bin_op, AsmOp::Custom("AND", 2)); - define_builtin_func("^_|=_", arith_bin_op, AsmOp::Custom("OR", 2)); - define_builtin_func("^_^=_", arith_bin_op, AsmOp::Custom("XOR", 2)); + define_builtin_func("^_&=_", arith_bin_op, compile_and); + define_builtin_func("^_|=_", arith_bin_op, compile_or); + define_builtin_func("^_^=_", arith_bin_op, compile_xor); define_builtin_func("muldiv", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, -1)); define_builtin_func("muldivr", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, 0)); define_builtin_func("muldivc", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, 1)); @@ -1063,6 +1184,8 @@ void define_builtins() { AsmOp::Nop()); define_builtin_func("~dump", TypeExpr::new_forall({X}, TypeExpr::new_map(X, TypeExpr::new_tensor({X, Unit}))), AsmOp::Custom("s0 DUMP", 1, 1), true); + define_builtin_func("~strdump", TypeExpr::new_forall({X}, TypeExpr::new_map(X, TypeExpr::new_tensor({X, Unit}))), + AsmOp::Custom("STRDUMP", 1, 1), true); define_builtin_func("run_method0", TypeExpr::new_map(Int, Unit), [](auto a, auto b, auto c) { return compile_run_method(a, b, c, 0, false); }, true); define_builtin_func("run_method1", TypeExpr::new_forall({X}, TypeExpr::new_map(TypeExpr::new_tensor({Int, X}), Unit)), diff --git a/crypto/func/func-main.cpp b/crypto/func/func-main.cpp new file mode 100644 index 00000000..45194ea3 --- /dev/null +++ b/crypto/func/func-main.cpp @@ -0,0 +1,127 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "func.h" +#include "parser/srcread.h" +#include "parser/lexer.h" +#include "parser/symtable.h" +#include +#include +#include "git.h" + +void usage(const char* progname) { + std::cerr + << "usage: " << progname + << " [-vIAPSR][-O][-i][-o][-W] { ...}\n" + "\tGenerates Fift TVM assembler code from a funC source\n" + "-I\tEnables interactive mode (parse stdin)\n" + "-o\tWrites generated code into specified file instead of stdout\n" + "-v\tIncreases verbosity level (extra information output into stderr)\n" + "-i\tSets indentation for the output code (in two-space units)\n" + "-A\tPrefix code with `\"Asm.fif\" include` preamble\n" + "-O\tSets optimization level (2 by default)\n" + "-P\tEnvelope code into PROGRAM{ ... }END>c\n" + "-S\tInclude stack layout comments in the output code\n" + "-R\tInclude operation rewrite comments in the output code\n" + "-W\tInclude Fift code to serialize and save generated code into specified BoC file. Enables " + "-A and -P.\n" + "\t-s\tOutput semantic version of FunC and exit\n" + "\t-V\tShow func build information\n"; + std::exit(2); +} + +int main(int argc, char* const argv[]) { + int i; + std::string output_filename; + while ((i = getopt(argc, argv, "Ahi:Io:O:PRsSvW:V")) != -1) { + switch (i) { + case 'A': + funC::asm_preamble = true; + break; + case 'I': + funC::interactive = true; + break; + case 'i': + funC::indent = std::max(0, atoi(optarg)); + break; + case 'o': + output_filename = optarg; + break; + case 'O': + funC::opt_level = std::max(0, atoi(optarg)); + break; + case 'P': + funC::program_envelope = true; + break; + case 'R': + funC::op_rewrite_comments = true; + break; + case 'S': + funC::stack_layout_comments = true; + break; + case 'v': + ++funC::verbosity; + break; + case 'W': + funC::boc_output_filename = optarg; + funC::asm_preamble = funC::program_envelope = true; + break; + case 's': + std::cout << funC::func_version << "\n"; + std::exit(0); + break; + case 'V': + std::cout << "FunC semantic version: v" << funC::func_version << "\n"; + std::cout << "Build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); + break; + case 'h': + default: + usage(argv[0]); + } + } + + std::ostream *outs = &std::cout; + + std::unique_ptr fs; + if (!output_filename.empty()) { + fs = std::make_unique(output_filename, fs->trunc | fs->out); + if (!fs->is_open()) { + std::cerr << "failed to create output file " << output_filename << '\n'; + return 2; + } + outs = fs.get(); + } + + std::vector sources; + + while (optind < argc) { + sources.push_back(std::string(argv[optind++])); + } + + return funC::func_proceed(sources, *outs, std::cerr); +} diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index b5d769cb..3daac5d7 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -28,16 +28,14 @@ #include "func.h" #include "parser/srcread.h" #include "parser/lexer.h" -#include "parser/symtable.h" #include -#include #include "git.h" namespace funC { int verbosity, indent, opt_level = 2; bool stack_layout_comments, op_rewrite_comments, program_envelope, asm_preamble; -std::ostream* outs = &std::cout; +bool interactive = false; std::string generated_from, boc_output_filename; /* @@ -46,58 +44,59 @@ std::string generated_from, boc_output_filename; * */ -void generate_output_func(SymDef* func_sym) { +void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &errs) { SymValCodeFunc* func_val = dynamic_cast(func_sym->value); assert(func_val); std::string name = sym::symbols.get_name(func_sym->sym_idx); if (verbosity >= 2) { - std::cerr << "\n\n=========================\nfunction " << name << " : " << func_val->get_type() << std::endl; + errs << "\n\n=========================\nfunction " << name << " : " << func_val->get_type() << std::endl; } if (!func_val->code) { - std::cerr << "( function `" << name << "` undefined )\n"; + errs << "( function `" << name << "` undefined )\n"; + throw src::ParseError(func_sym->loc, name); } else { CodeBlob& code = *(func_val->code); if (verbosity >= 3) { - code.print(std::cerr, 9); + code.print(errs, 9); } code.simplify_var_types(); if (verbosity >= 5) { - std::cerr << "after simplify_var_types: \n"; - code.print(std::cerr, 0); + errs << "after simplify_var_types: \n"; + code.print(errs, 0); } code.prune_unreachable_code(); if (verbosity >= 5) { - std::cerr << "after prune_unreachable: \n"; - code.print(std::cerr, 0); + errs << "after prune_unreachable: \n"; + code.print(errs, 0); } code.split_vars(true); if (verbosity >= 5) { - std::cerr << "after split_vars: \n"; - code.print(std::cerr, 0); + errs << "after split_vars: \n"; + code.print(errs, 0); } for (int i = 0; i < 8; i++) { code.compute_used_code_vars(); if (verbosity >= 4) { - std::cerr << "after compute_used_vars: \n"; - code.print(std::cerr, 6); + errs << "after compute_used_vars: \n"; + code.print(errs, 6); } code.fwd_analyze(); if (verbosity >= 5) { - std::cerr << "after fwd_analyze: \n"; - code.print(std::cerr, 6); + errs << "after fwd_analyze: \n"; + code.print(errs, 6); } code.prune_unreachable_code(); if (verbosity >= 5) { - std::cerr << "after prune_unreachable: \n"; - code.print(std::cerr, 6); + errs << "after prune_unreachable: \n"; + code.print(errs, 6); } } code.mark_noreturn(); if (verbosity >= 3) { - code.print(std::cerr, 15); + code.print(errs, 15); } if (verbosity >= 2) { - std::cerr << "\n---------- resulting code for " << name << " -------------\n"; + errs << "\n---------- resulting code for " << name << " -------------\n"; } bool inline_func = (func_val->flags & 1); bool inline_ref = (func_val->flags & 2); @@ -107,7 +106,7 @@ void generate_output_func(SymDef* func_sym) { } else if (inline_ref) { modifier = "REF"; } - *outs << std::string(indent * 2, ' ') << name << " PROC" << modifier << ":<{\n"; + outs << std::string(indent * 2, ' ') << name << " PROC" << modifier << ":<{\n"; int mode = 0; if (stack_layout_comments) { mode |= Stack::_StkCmt | Stack::_CptStkCmt; @@ -120,145 +119,71 @@ void generate_output_func(SymDef* func_sym) { if (fv && (fv->flags & 1) && code.ops->noreturn()) { mode |= Stack::_InlineFunc; } - code.generate_code(*outs, mode, indent + 1); - *outs << std::string(indent * 2, ' ') << "}>\n"; + code.generate_code(outs, mode, indent + 1); + outs << std::string(indent * 2, ' ') << "}>\n"; if (verbosity >= 2) { - std::cerr << "--------------\n"; + errs << "--------------\n"; } } } -int generate_output() { +int generate_output(std::ostream &outs, std::ostream &errs) { if (asm_preamble) { - *outs << "\"Asm.fif\" include\n"; + outs << "\"Asm.fif\" include\n"; } - *outs << "// automatically generated from " << generated_from << std::endl; + outs << "// automatically generated from " << generated_from << std::endl; if (program_envelope) { - *outs << "PROGRAM{\n"; + outs << "PROGRAM{\n"; } for (SymDef* func_sym : glob_func) { SymValCodeFunc* func_val = dynamic_cast(func_sym->value); assert(func_val); std::string name = sym::symbols.get_name(func_sym->sym_idx); - *outs << std::string(indent * 2, ' '); + outs << std::string(indent * 2, ' '); if (func_val->method_id.is_null()) { - *outs << "DECLPROC " << name << "\n"; + outs << "DECLPROC " << name << "\n"; } else { - *outs << func_val->method_id << " DECLMETHOD " << name << "\n"; + outs << func_val->method_id << " DECLMETHOD " << name << "\n"; } } for (SymDef* gvar_sym : glob_vars) { assert(dynamic_cast(gvar_sym->value)); std::string name = sym::symbols.get_name(gvar_sym->sym_idx); - *outs << std::string(indent * 2, ' ') << "DECLGLOBVAR " << name << "\n"; + outs << std::string(indent * 2, ' ') << "DECLGLOBVAR " << name << "\n"; } int errors = 0; for (SymDef* func_sym : glob_func) { try { - generate_output_func(func_sym); + generate_output_func(func_sym, outs, errs); } catch (src::Error& err) { - std::cerr << "cannot generate code for function `" << sym::symbols.get_name(func_sym->sym_idx) << "`:\n" + errs << "cannot generate code for function `" << sym::symbols.get_name(func_sym->sym_idx) << "`:\n" << err << std::endl; ++errors; } } if (program_envelope) { - *outs << "}END>c\n"; + outs << "}END>c\n"; } if (!boc_output_filename.empty()) { - *outs << "2 boc+>B \"" << boc_output_filename << "\" B>file\n"; + outs << "2 boc+>B \"" << boc_output_filename << "\" B>file\n"; } return errors; } -} // namespace funC - -void usage(const char* progname) { - std::cerr - << "usage: " << progname - << " [-vIAPSR][-O][-i][-o][-W] { ...}\n" - "\tGenerates Fift TVM assembler code from a funC source\n" - "-I\tEnables interactive mode (parse stdin)\n" - "-o\tWrites generated code into specified file instead of stdout\n" - "-v\tIncreases verbosity level (extra information output into stderr)\n" - "-i\tSets indentation for the output code (in two-space units)\n" - "-A\tPrefix code with `\"Asm.fif\" include` preamble\n" - "-O\tSets optimization level (2 by default)\n" - "-P\tEnvelope code into PROGRAM{ ... }END>c\n" - "-S\tInclude stack layout comments in the output code\n" - "-R\tInclude operation rewrite comments in the output code\n" - "-W\tInclude Fift code to serialize and save generated code into specified BoC file. Enables " - "-A and -P.\n" - "\t-s\tOutput semantic version of FunC and exit\n" - "\t-V\tShow func build information\n"; - std::exit(2); -} - -void output_inclusion_stack() { +void output_inclusion_stack(std::ostream &errs) { while (!funC::inclusion_locations.empty()) { src::SrcLocation loc = funC::inclusion_locations.top(); funC::inclusion_locations.pop(); if (loc.fdescr) { - std::cerr << "note: included from "; - loc.show(std::cerr); - std::cerr << std::endl; + errs << "note: included from "; + loc.show(errs); + errs << std::endl; } } } -std::string output_filename; - -int main(int argc, char* const argv[]) { - int i; - bool interactive = false; - while ((i = getopt(argc, argv, "Ahi:Io:O:PRsSvW:V")) != -1) { - switch (i) { - case 'A': - funC::asm_preamble = true; - break; - case 'I': - interactive = true; - break; - case 'i': - funC::indent = std::max(0, atoi(optarg)); - break; - case 'o': - output_filename = optarg; - break; - case 'O': - funC::opt_level = std::max(0, atoi(optarg)); - break; - case 'P': - funC::program_envelope = true; - break; - case 'R': - funC::op_rewrite_comments = true; - break; - case 'S': - funC::stack_layout_comments = true; - break; - case 'v': - ++funC::verbosity; - break; - case 'W': - funC::boc_output_filename = optarg; - funC::asm_preamble = funC::program_envelope = true; - break; - case 's': - std::cout << funC::func_version << "\n"; - std::exit(0); - break; - case 'V': - std::cout << "FunC semantic version: v" << funC::func_version << "\n"; - std::cout << "Build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; - std::exit(0); - break; - case 'h': - default: - usage(argv[0]); - } - } +int func_proceed(const std::vector &sources, std::ostream &outs, std::ostream &errs) { if (funC::program_envelope && !funC::indent) { funC::indent = 1; } @@ -268,12 +193,11 @@ int main(int argc, char* const argv[]) { int ok = 0, proc = 0; try { - while (optind < argc) { - // funC::generated_from += std::string{"`"} + argv[optind] + "` "; - ok += funC::parse_source_file(argv[optind++]); + for (auto src : sources) { + ok += funC::parse_source_file(src.c_str()); proc++; } - if (interactive) { + if (funC::interactive) { funC::generated_from += "stdin "; ok += funC::parse_source_stdin(); proc++; @@ -284,29 +208,24 @@ int main(int argc, char* const argv[]) { if (!proc) { throw src::Fatal{"no source files, no output"}; } - std::unique_ptr fs; - if (!output_filename.empty()) { - fs = std::make_unique(output_filename, fs->trunc | fs->out); - if (!fs->is_open()) { - std::cerr << "failed to create output file " << output_filename << '\n'; - return 2; - } - funC::outs = fs.get(); - } - funC::generate_output(); + return funC::generate_output(outs, errs); } catch (src::Fatal& fatal) { - std::cerr << "fatal: " << fatal << std::endl; - output_inclusion_stack(); - std::exit(1); + errs << "fatal: " << fatal << std::endl; + output_inclusion_stack(errs); + return 2; } catch (src::Error& error) { - std::cerr << error << std::endl; - output_inclusion_stack(); - std::exit(1); + errs << error << std::endl; + output_inclusion_stack(errs); + return 2; } catch (funC::UnifyError& unif_err) { - std::cerr << "fatal: "; - unif_err.print_message(std::cerr); - std::cerr << std::endl; - output_inclusion_stack(); - std::exit(1); + errs << "fatal: "; + unif_err.print_message(errs); + errs << std::endl; + output_inclusion_stack(errs); + return 2; } + + return 0; } + +} // namespace funC \ No newline at end of file diff --git a/crypto/func/func.h b/crypto/func/func.h index 886a21ed..5db99725 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -39,7 +39,7 @@ extern std::string generated_from; constexpr int optimize_depth = 20; -const std::string func_version{"0.2.0"}; +const std::string func_version{"0.3.0"}; enum Keyword { _Eof = -1, @@ -1631,6 +1631,7 @@ inline compile_func_t make_ext_compile(AsmOp op) { struct SymValAsmFunc : SymValFunc { simple_compile_func_t simple_compile; compile_func_t ext_compile; + td::uint64 crc; ~SymValAsmFunc() override = default; SymValAsmFunc(TypeExpr* ft, const AsmOp& _macro, bool impure = false) : SymValFunc(-1, ft, impure), simple_compile(make_simple_compile(_macro)) { @@ -1665,4 +1666,19 @@ AsmOp push_const(td::RefInt256 x); void define_builtins(); + +extern int verbosity, indent, opt_level; +extern bool stack_layout_comments, op_rewrite_comments, program_envelope, asm_preamble, interactive; +extern std::string generated_from, boc_output_filename; + +/* + * + * OUTPUT CODE GENERATOR + * + */ + +int func_proceed(const std::vector &sources, std::ostream &outs, std::ostream &errs); + } // namespace funC + + diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 0d2aa985..92dfe47b 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -254,9 +254,7 @@ void parse_const_decl(Lexer& lex) { if (!sym_def) { lex.cur().error_at("cannot define global symbol `", "`"); } - if (sym_def->value) { - lex.cur().error_at("global symbol `", "` already exists"); - } + Lexem ident = lex.cur(); lex.next(); if (lex.tp() != '=') { lex.cur().error_at("expected = instead of ", ""); @@ -273,10 +271,11 @@ void parse_const_decl(Lexer& lex) { if ((wanted_type != Expr::_None) && (x->cls != wanted_type)) { lex.cur().error("expression type does not match wanted type"); } + SymValConst* new_value = nullptr; if (x->cls == Expr::_Const) { // Integer constant - sym_def->value = new SymValConst{const_cnt++, x->intval}; + new_value = new SymValConst{const_cnt++, x->intval}; } else if (x->cls == Expr::_SliceConst) { // Slice constant (string) - sym_def->value = new SymValConst{const_cnt++, x->strval}; + new_value = new SymValConst{const_cnt++, x->strval}; } else if (x->cls == Expr::_Apply) { code.emplace_back(loc, Op::_Import, std::vector()); auto tmp_vars = x->pre_compile(code); @@ -304,10 +303,20 @@ void parse_const_decl(Lexer& lex) { if (op.origin.is_null() || !op.origin->is_valid()) { lex.cur().error("precompiled expression did not result in a valid integer constant"); } - sym_def->value = new SymValConst{const_cnt++, op.origin}; + new_value = new SymValConst{const_cnt++, op.origin}; } else { lex.cur().error("integer or slice literal or constant expected"); } + if (sym_def->value) { + SymValConst* old_value = dynamic_cast(sym_def->value); + Keyword new_type = new_value->get_type(); + if (!old_value || old_value->get_type() != new_type || + (new_type == _Int && *old_value->get_int_value() != *new_value->get_int_value()) || + (new_type == _Slice && old_value->get_str_value() != new_value->get_str_value())) { + ident.error_at("global symbol `", "` already exists"); + } + } + sym_def->value = new_value; } FormalArgList parse_formal_args(Lexer& lex) { @@ -1261,19 +1270,48 @@ SymValAsmFunc* parse_asm_func_body(Lexer& lex, TypeExpr* func_type, const Formal lex.expect(')'); } while (lex.tp() == _String) { - asm_ops.push_back(AsmOp::Parse(lex.cur().str, cnt, width)); - lex.next(); - if (asm_ops.back().is_custom()) { - cnt = width; + std::string ops = lex.cur().str; // \n\n... + std::string op; + for (const char& c : ops) { + if (c == '\n') { + if (!op.empty()) { + asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + if (asm_ops.back().is_custom()) { + cnt = width; + } + op.clear(); + } + } else { + op.push_back(c); + } } + if (!op.empty()) { + asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + if (asm_ops.back().is_custom()) { + cnt = width; + } + } + lex.next(); } if (asm_ops.empty()) { throw src::ParseError{lex.cur().loc, "string with assembler instruction expected"}; } lex.expect(';'); + std::string crc_s; + for (const AsmOp& asm_op : asm_ops) { + crc_s += asm_op.op; + } + crc_s.push_back(impure); + for (const int& x : arg_order) { + crc_s += std::string((const char*) (&x), (const char*) (&x + 1)); + } + for (const int& x : ret_order) { + crc_s += std::string((const char*) (&x), (const char*) (&x + 1)); + } auto res = new SymValAsmFunc{func_type, asm_ops, impure}; res->arg_order = std::move(arg_order); res->ret_order = std::move(ret_order); + res->crc = td::crc64(crc_s); return res; } @@ -1439,16 +1477,22 @@ void parse_func_def(Lexer& lex) { // code->print(std::cerr); // !!!DEBUG!!! func_sym_code->code = code; } else { + Lexem asm_lexem = lex.cur(); + SymValAsmFunc* asm_func = parse_asm_func_body(lex, func_type, arg_list, ret_type, impure); if (func_sym_val) { if (dynamic_cast(func_sym_val)) { - lex.cur().error("function `"s + func_name.str + "` was already declared as an ordinary function"); + asm_lexem.error("function `"s + func_name.str + "` was already declared as an ordinary function"); } - if (dynamic_cast(func_sym_val)) { - lex.cur().error("redefinition of built-in assembler function `"s + func_name.str + "`"); + SymValAsmFunc* asm_func_old = dynamic_cast(func_sym_val); + if (asm_func_old) { + if (asm_func->crc != asm_func_old->crc) { + asm_lexem.error("redefinition of built-in assembler function `"s + func_name.str + "`"); + } + } else { + asm_lexem.error("redefinition of previously (somehow) defined function `"s + func_name.str + "`"); } - lex.cur().error("redefinition of previously (somehow) defined function `"s + func_name.str + "`"); } - func_sym->value = parse_asm_func_body(lex, func_type, arg_list, ret_type, impure); + func_sym->value = asm_func; } if (method_id.not_null()) { auto val = dynamic_cast(func_sym->value); @@ -1657,7 +1701,14 @@ bool parse_source_file(const char* filename, src::Lexem lex) { throw src::Fatal{msg}; } } - std::string real_filename = td::realpath(td::CSlice(filename)).move_as_ok(); + + auto path_res = td::realpath(td::CSlice(filename)); + if (path_res.is_error()) { + auto error = path_res.move_as_error(); + lex.error(error.message().c_str()); + return false; + } + std::string real_filename = path_res.move_as_ok(); if (std::count(source_files.begin(), source_files.end(), real_filename)) { if (verbosity >= 2) { if (lex.tp) { diff --git a/crypto/func/test/co2.fc b/crypto/func/test/co2.fc new file mode 100644 index 00000000..f5fcb748 --- /dev/null +++ b/crypto/func/test/co2.fc @@ -0,0 +1,15 @@ +const int x = 5; +const slice s = "abacaba"; +const int y = 3; +const slice s = "abacaba"; +const int x = 5; +const int z = 4, z = 4; + +int sdeq (slice s1, slice s2) asm "SDEQ"; + +() main() { + throw_unless(101, x == 5); + throw_unless(102, y == 3); + throw_unless(103, z == 4); + throw_unless(104, sdeq(s, "abacaba")); +} diff --git a/crypto/func/test/co3.fc b/crypto/func/test/co3.fc new file mode 100644 index 00000000..398592d3 --- /dev/null +++ b/crypto/func/test/co3.fc @@ -0,0 +1,24 @@ +const val1 = 123456789; +const val2 = 987654321; +const val3 = 135792468; +const val4 = 246813579; + +const prec_and = val1 & val2; +const prec_or = val1 | val2; +const prec_xor = val1 ^ val2; +const prec_logic = ((val1 & val2) | val3) ^ val4; +const prec_nand = val1 & (~ val2); + +int get_and() { return prec_and; } +int get_or() { return prec_or; } +int get_xor() { return prec_xor; } +int get_logic() { return prec_logic; } +int get_nand() { return prec_nand; } + +_ main() { + throw_unless(101, get_and() == 39471121); + throw_unless(102, get_or() == 1071639989); + throw_unless(103, get_xor() == 1032168868); + throw_unless(104, get_logic() == 82599134); + throw_unless(105, get_nand() == 83985668); +} diff --git a/crypto/func/test/s2.fc b/crypto/func/test/s2.fc new file mode 100644 index 00000000..c6df49d5 --- /dev/null +++ b/crypto/func/test/s2.fc @@ -0,0 +1,26 @@ +slice test1() asm """ + "Test" $>s + PUSHSLICE +"""; + +slice test2() asm """ + "Hello" + " " + "World" + $+ $+ $>s + PUSHSLICE +"""; + +int sdeq (slice s1, slice s2) asm """SDEQ"""; +int sdeq (slice s1, slice s2) asm "SDEQ" ""; +int sdeq (slice s1, slice s2) asm "" """ +SDEQ +"""; + +() main() { + slice s = test1(); + throw_unless(101, sdeq(s, "Test")); + + slice s = test2(); + throw_unless(102, sdeq(s, "Hello World")); +} diff --git a/crypto/funcfiftlib/funcfiftlib.cpp b/crypto/funcfiftlib/funcfiftlib.cpp new file mode 100644 index 00000000..6c8912bc --- /dev/null +++ b/crypto/funcfiftlib/funcfiftlib.cpp @@ -0,0 +1,131 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "func/func.h" +#include "git.h" +#include "td/utils/JsonBuilder.h" +#include "fift/utils.h" +#include "td/utils/base64.h" +#include +#include + +std::string escape_json(const std::string &s) { + std::ostringstream o; + for (auto c = s.cbegin(); c != s.cend(); c++) { + switch (*c) { + case '"': o << "\\\""; break; + case '\\': o << "\\\\"; break; + case '\b': o << "\\b"; break; + case '\f': o << "\\f"; break; + case '\n': o << "\\n"; break; + case '\r': o << "\\r"; break; + case '\t': o << "\\t"; break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + o << "\\u" + << std::hex << std::setw(4) << std::setfill('0') << static_cast(*c); + } else { + o << *c; + } + } + } + return o.str(); +} + +td::Result compile_internal(char *config_json) { + TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json))) + auto &obj = input_json.get_object(); + + TRY_RESULT(opt_level, td::get_json_object_int_field(obj, "optLevel", false)); + TRY_RESULT(sources_obj, td::get_json_object_field(obj, "sources", td::JsonValue::Type::Array, false)); + + auto &sources_arr = sources_obj.get_array(); + + std::vector sources; + + for (auto &item : sources_arr) { + sources.push_back(item.get_string().str()); + } + + funC::opt_level = std::max(0, opt_level); + funC::program_envelope = true; + funC::verbosity = 0; + funC::indent = 1; + + std::ostringstream outs, errs; + auto compile_res = funC::func_proceed(sources, outs, errs); + + if (compile_res != 0) { + return td::Status::Error(std::string("Func compilation error: ") + errs.str()); + } + + TRY_RESULT(code_cell, fift::compile_asm(outs.str(), "/fiftlib/", false)); + TRY_RESULT(boc, vm::std_boc_serialize(code_cell)); + + td::JsonBuilder result_json; + auto result_obj = result_json.enter_object(); + result_obj("status", "ok"); + result_obj("codeBoc", td::base64_encode(boc)); + result_obj("fiftCode", escape_json(outs.str())); + result_obj.leave(); + + outs.clear(); + errs.clear(); + + return result_json.string_builder().as_cslice().str(); +} + +extern "C" { + +const char* version() { + auto version_json = td::JsonBuilder(); + auto obj = version_json.enter_object(); + obj("funcVersion", funC::func_version); + obj("funcFiftLibCommitHash", GitMetadata::CommitSHA1()); + obj("funcFiftLibCommitDate", GitMetadata::CommitDate()); + obj.leave(); + return strdup(version_json.string_builder().as_cslice().c_str()); +} + +const char *func_compile(char *config_json) { + auto res = compile_internal(config_json); + + if (res.is_error()) { + auto result = res.move_as_error(); + auto error_res = td::JsonBuilder(); + auto error_o = error_res.enter_object(); + error_o("status", "error"); + error_o("message", result.message().str()); + error_o.leave(); + return strdup(error_res.string_builder().as_cslice().c_str()); + } + + auto res_string = res.move_as_ok(); + + return strdup(res_string.c_str()); +} +} diff --git a/crypto/parser/lexer.cpp b/crypto/parser/lexer.cpp index 5c5b77d8..624d8dd2 100644 --- a/crypto/parser/lexer.cpp +++ b/crypto/parser/lexer.cpp @@ -125,8 +125,9 @@ int Lexem::set(std::string _str, const SrcLocation& _loc, int _tp, int _val) { } Lexer::Lexer(SourceReader& _src, bool init, std::string active_chars, std::string eol_cmts, std::string open_cmts, - std::string close_cmts, std::string quote_chars) - : src(_src), eof(false), lexem("", src.here(), Lexem::Undefined), peek_lexem("", {}, Lexem::Undefined) { + std::string close_cmts, std::string quote_chars, std::string multiline_quote) + : src(_src), eof(false), lexem("", src.here(), Lexem::Undefined), peek_lexem("", {}, Lexem::Undefined), + multiline_quote(std::move(multiline_quote)) { std::memset(char_class, 0, sizeof(char_class)); unsigned char activity = cc::active; for (char c : active_chars) { @@ -171,6 +172,19 @@ void Lexer::set_spec(std::array& arr, std::string setup) { } } +bool Lexer::is_multiline_quote(const char* begin, const char* end) { + if (multiline_quote.empty()) { + return false; + } + for (const char& c : multiline_quote) { + if (begin == end || *begin != c) { + return false; + } + ++begin; + } + return true; +} + void Lexer::expect(int exp_tp, const char* msg) { if (tp() != exp_tp) { throw ParseError{lexem.loc, (msg ? std::string{msg} : Lexem::lexem_name_str(exp_tp)) + " expected instead of " + @@ -234,6 +248,37 @@ const Lexem& Lexer::next() { } return lexem.clear(src.here(), Lexem::Eof); } + if (is_multiline_quote(src.get_ptr(), src.get_end_ptr())) { + src.advance(multiline_quote.size()); + const char* begin = src.get_ptr(); + const char* end = nullptr; + SrcLocation here = src.here(); + std::string body; + while (!src.is_eof()) { + if (src.is_eoln()) { + body.push_back('\n'); + src.load_line(); + continue; + } + if (is_multiline_quote(src.get_ptr(), src.get_end_ptr())) { + end = src.get_ptr(); + src.advance(multiline_quote.size()); + break; + } + body.push_back(src.cur_char()); + src.advance(1); + } + if (!end) { + src.error("string extends past end of file"); + } + lexem.set(body, here, Lexem::String); + int c = src.cur_char(); + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { + lexem.val = c; + src.advance(1); + } + return lexem; + } int c = src.cur_char(); const char* end = src.get_ptr(); if (is_quote_char(c) || c == '`') { diff --git a/crypto/parser/lexer.h b/crypto/parser/lexer.h index 12556e46..686d8eac 100644 --- a/crypto/parser/lexer.h +++ b/crypto/parser/lexer.h @@ -71,6 +71,7 @@ class Lexer { Lexem lexem, peek_lexem; unsigned char char_class[128]; std::array eol_cmt, cmt_op, cmt_cl; + std::string multiline_quote; enum cc { left_active = 2, right_active = 1, active = 3, allow_repeat = 4, quote_char = 8 }; public: @@ -78,7 +79,8 @@ class Lexer { return eof; } Lexer(SourceReader& _src, bool init = false, std::string active_chars = ";,() ~.", std::string eol_cmts = ";;", - std::string open_cmts = "{-", std::string close_cmts = "-}", std::string quote_chars = "\""); + std::string open_cmts = "{-", std::string close_cmts = "-}", std::string quote_chars = "\"", + std::string multiline_quote = "\"\"\""); const Lexem& next(); const Lexem& cur() const { return lexem; @@ -109,6 +111,7 @@ class Lexer { private: void set_spec(std::array& arr, std::string setup); + bool is_multiline_quote(const char* begin, const char* end); }; } // namespace src diff --git a/crypto/smc-envelope/ManualDns.cpp b/crypto/smc-envelope/ManualDns.cpp index 8f5097c0..42d379c2 100644 --- a/crypto/smc-envelope/ManualDns.cpp +++ b/crypto/smc-envelope/ManualDns.cpp @@ -433,8 +433,12 @@ td::Result> ManualDns::resolve_raw_or_throw(td: } else { if (category.is_zero()) { vm::Dictionary dict(std::move(data), 256); - dict.check_for_each([&](auto cs, td::ConstBitPtr key, int n) { + dict.check_for_each([&](td::Ref cs, td::ConstBitPtr key, int n) { CHECK(n == 256); + if (cs.is_null() || cs->size_ext() != 0x10000) { + return true; + } + cs = vm::load_cell_slice_ref(cs->prefetch_ref()); vec.push_back({name.str(), td::Bits256(key), cs}); return true; }); diff --git a/crypto/tl/tlbc.cpp b/crypto/tl/tlbc.cpp index 127eacb0..01b8a31a 100644 --- a/crypto/tl/tlbc.cpp +++ b/crypto/tl/tlbc.cpp @@ -2423,7 +2423,7 @@ std::vector source_fdescr; bool parse_source(std::istream* is, src::FileDescr* fdescr) { src::SourceReader reader{is, fdescr}; - src::Lexer lex{reader, true, "(){}:;? #$. ^~ #", "//", "/*", "*/"}; + src::Lexer lex{reader, true, "(){}:;? #$. ^~ #", "//", "/*", "*/", ""}; while (lex.tp() != src::_Eof) { parse_constructor_def(lex); // std::cerr << lex.cur().str << '\t' << lex.cur().name_str() << std::endl; diff --git a/crypto/tl/tlblib.cpp b/crypto/tl/tlblib.cpp index d0c1b538..0e0e5626 100644 --- a/crypto/tl/tlblib.cpp +++ b/crypto/tl/tlblib.cpp @@ -125,8 +125,11 @@ bool TupleT::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { } bool TLB::validate_ref_internal(int* ops, Ref cell_ref, bool weak) const { - if (ops && --*ops < 0) { - return false; + if (ops) { + if (*ops <= 0) { + return false; + } + --*ops; } bool is_special; auto cs = load_cell_slice_special(std::move(cell_ref), is_special); diff --git a/crypto/vm/cells/CellBuilder.cpp b/crypto/vm/cells/CellBuilder.cpp index 481c2581..43058531 100644 --- a/crypto/vm/cells/CellBuilder.cpp +++ b/crypto/vm/cells/CellBuilder.cpp @@ -335,7 +335,7 @@ bool CellBuilder::store_ulong_rchk_bool(unsigned long long val, unsigned val_bit } CellBuilder& CellBuilder::store_long(long long val, unsigned val_bits) { - return store_long_top(val << (64 - val_bits), val_bits); + return store_long_top(val_bits == 0 ? 0 : (unsigned long long)val << (64 - val_bits), val_bits); } CellBuilder& CellBuilder::store_long_top(unsigned long long val, unsigned top_bits) { diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 90b78558..e1df5759 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -1055,44 +1055,47 @@ std::ostream& operator<<(std::ostream& os, Ref cs_ref) { // If can_be_special is not null, then it is allowed to load special cell // Flag whether loaded cell is actually special will be stored into can_be_special -VirtualCell::LoadedCell load_cell_slice_impl(const Ref& cell, bool* can_be_special) { - auto* vm_state_interface = VmStateInterface::get(); - if (vm_state_interface) { - vm_state_interface->register_cell_load(cell->get_hash()); - } - auto r_loaded_cell = cell->load_cell(); - if (r_loaded_cell.is_error()) { - throw VmError{Excno::cell_und, "failed to load cell"}; - } - auto loaded_cell = r_loaded_cell.move_as_ok(); - if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::PrunnedBranch) { - auto virtualization = loaded_cell.virt.get_virtualization(); - if (virtualization != 0) { - throw VmVirtError{virtualization}; +VirtualCell::LoadedCell load_cell_slice_impl(Ref cell, bool* can_be_special) { + while (true) { + auto* vm_state_interface = VmStateInterface::get(); + if (vm_state_interface) { + vm_state_interface->register_cell_load(cell->get_hash()); } - } - if (can_be_special) { - *can_be_special = loaded_cell.data_cell->is_special(); - } else if (loaded_cell.data_cell->is_special()) { - if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::Library) { - if (vm_state_interface) { - CellSlice cs(std::move(loaded_cell)); - DCHECK(cs.size() == Cell::hash_bits + 8); - auto library_cell = vm_state_interface->load_library(cs.data_bits() + 8); - if (library_cell.not_null()) { - //TODO: fix infinity loop - return load_cell_slice_impl(library_cell, nullptr); - } - throw VmError{Excno::cell_und, "failed to load library cell"}; + auto r_loaded_cell = cell->load_cell(); + if (r_loaded_cell.is_error()) { + throw VmError{Excno::cell_und, "failed to load cell"}; + } + auto loaded_cell = r_loaded_cell.move_as_ok(); + if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::PrunnedBranch) { + auto virtualization = loaded_cell.virt.get_virtualization(); + if (virtualization != 0) { + throw VmVirtError{virtualization}; } - throw VmError{Excno::cell_und, "failed to load library cell (no vm_state_interface available)"}; - } else if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::PrunnedBranch) { - CHECK(loaded_cell.virt.get_virtualization() == 0); - throw VmError{Excno::cell_und, "trying to load prunned cell"}; } - throw VmError{Excno::cell_und, "unexpected special cell"}; + if (can_be_special) { + *can_be_special = loaded_cell.data_cell->is_special(); + } else if (loaded_cell.data_cell->is_special()) { + if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::Library) { + if (vm_state_interface) { + CellSlice cs(std::move(loaded_cell)); + DCHECK(cs.size() == Cell::hash_bits + 8); + auto library_cell = vm_state_interface->load_library(cs.data_bits() + 8); + if (library_cell.not_null()) { + cell = library_cell; + can_be_special = nullptr; + continue; + } + throw VmError{Excno::cell_und, "failed to load library cell"}; + } + throw VmError{Excno::cell_und, "failed to load library cell (no vm_state_interface available)"}; + } else if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::PrunnedBranch) { + CHECK(loaded_cell.virt.get_virtualization() == 0); + throw VmError{Excno::cell_und, "trying to load prunned cell"}; + } + throw VmError{Excno::cell_und, "unexpected special cell"}; + } + return loaded_cell; } - return loaded_cell; } CellSlice load_cell_slice(const Ref& cell) { diff --git a/crypto/vm/debugops.cpp b/crypto/vm/debugops.cpp index 0ec205a3..3f27de23 100644 --- a/crypto/vm/debugops.cpp +++ b/crypto/vm/debugops.cpp @@ -105,6 +105,43 @@ int exec_dump_value(VmState* st, unsigned arg) { return 0; } +int exec_dump_string(VmState* st) { + VM_LOG(st) << "execute STRDUMP"; + if (!vm_debug_enabled) { + return 0; + } + + Stack& stack = st->get_stack(); + + if (stack.depth() > 0){ + auto cs = stack[0].as_slice(); + + if (cs.not_null()) { // wanted t_slice + auto size = cs->size(); + + if (size % 8 == 0) { + auto cnt = size / 8; + + unsigned char tmp[128]; + cs.write().fetch_bytes(tmp, cnt); + std::string s{tmp, tmp + cnt}; + + std::cerr << "#DEBUG#: " << s << std::endl; + } + else { + std::cerr << "#DEBUG#: slice contains not valid bits count" << std::endl; + } + + } else { + std::cerr << "#DEBUG#: is not a slice" << std::endl; + } + } else { + std::cerr << "#DEBUG#: s0 is absent" << std::endl; + } + + return 0; +} + void register_debug_ops(OpcodeTable& cp0) { using namespace std::placeholders; if (!vm_debug_enabled) { @@ -113,7 +150,9 @@ void register_debug_ops(OpcodeTable& cp0) { } else { // NB: all non-redefined opcodes in fe00..feff should be redirected to dummy debug definitions cp0.insert(OpcodeInstr::mksimple(0xfe00, 16, "DUMPSTK", exec_dump_stack)) - .insert(OpcodeInstr::mkfixedrange(0xfe01, 0xfe20, 16, 8, instr::dump_1c_and(0xff, "DEBUG "), exec_dummy_debug)) + .insert(OpcodeInstr::mkfixedrange(0xfe01, 0xfe14, 16, 8, instr::dump_1c_and(0xff, "DEBUG "), exec_dummy_debug)) + .insert(OpcodeInstr::mksimple(0xfe14, 16,"STRDUMP", exec_dump_string)) + .insert(OpcodeInstr::mkfixedrange(0xfe15, 0xfe20, 16, 8, instr::dump_1c_and(0xff, "DEBUG "), exec_dummy_debug)) .insert(OpcodeInstr::mkfixed(0xfe2, 12, 4, instr::dump_1sr("DUMP"), exec_dump_value)) .insert(OpcodeInstr::mkfixedrange(0xfe30, 0xfef0, 16, 8, instr::dump_1c_and(0xff, "DEBUG "), exec_dummy_debug)) .insert(OpcodeInstr::mkext(0xfef, 12, 4, dump_dummy_debug_str, exec_dummy_debug_str, compute_len_debug_str)); diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index 765bd9a6..fe16b767 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -313,7 +313,7 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { return td::Status::Error("bag of cells is too large"); } - boc_writers::FileWriter writer{fd, info.total_size}; + boc_writers::FileWriter writer{fd, (size_t) info.total_size}; auto store_ref = [&](unsigned long long value) { writer.store_uint(value, info.ref_byte_size); }; diff --git a/crypto/vm/stack.cpp b/crypto/vm/stack.cpp index c580a3d0..9a17c8e2 100644 --- a/crypto/vm/stack.cpp +++ b/crypto/vm/stack.cpp @@ -90,15 +90,27 @@ void StackEntry::dump(std::ostream& os) const { os << dec_string(as_int()); break; case t_cell: - os << "C{" << static_cast>(ref)->get_hash().to_hex() << "}"; + if (ref.not_null()) { + os << "C{" << static_cast>(ref)->get_hash().to_hex() << "}"; + } else { + os << "C{null}"; + } break; case t_builder: - os << "BC{" << static_cast>(ref)->to_hex() << "}"; + if (ref.not_null()) { + os << "BC{" << static_cast>(ref)->to_hex() << "}"; + } else { + os << "BC{null}"; + } break; case t_slice: { - os << "CS{"; - static_cast>(ref)->dump(os, 1, false); - os << '}'; + if (ref.not_null()) { + os << "CS{"; + static_cast>(ref)->dump(os, 1, false); + os << '}'; + } else { + os << "CS{null}"; + } break; } case t_string: diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 6a4cceaa..a164df83 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -474,7 +474,7 @@ int exec_store_var_integer(VmState* st, int len_bits, bool sgnd, bool quiet) { stack.check_underflow(2); auto x = stack.pop_int(); auto cbr = stack.pop_builder(); - unsigned len = ((x->bit_size(sgnd) + 7) >> 3); + unsigned len = (((unsigned)x->bit_size(sgnd) + 7) >> 3); if (len >= (1u << len_bits)) { throw VmError{Excno::range_chk}; } diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index fb2f4d1e..6668c6d5 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -503,7 +503,7 @@ int VmState::run() { bool VmState::try_commit() { if (cr.d[0].not_null() && cr.d[1].not_null() && cr.d[0]->get_depth() <= max_data_depth && - cr.d[1]->get_depth() <= max_data_depth) { + cr.d[1]->get_depth() <= max_data_depth && cr.d[0]->get_level() == 0 && cr.d[1]->get_level() == 0) { cstate.c4 = cr.d[0]; cstate.c5 = cr.d[1]; cstate.committed = true; diff --git a/fec/fec.cpp b/fec/fec.cpp index b194a173..102df038 100644 --- a/fec/fec.cpp +++ b/fec/fec.cpp @@ -19,6 +19,7 @@ #include "fec.h" #include "td/utils/overloaded.h" #include "auto/tl/ton_api.hpp" +#include "td/utils/misc.h" namespace ton { @@ -98,24 +99,37 @@ td::uint32 FecType::symbol_size() const { } td::Result FecType::create(tl_object_ptr obj) { + td::int32 data_size_int, symbol_size_int, symbols_count_int; + ton_api::downcast_call(*obj, td::overloaded([&](const auto &obj) { + data_size_int = obj.data_size_; + symbol_size_int = obj.symbol_size_; + symbols_count_int = obj.symbols_count_; + })); + TRY_RESULT(data_size, td::narrow_cast_safe(data_size_int)); + TRY_RESULT(symbol_size, td::narrow_cast_safe(symbol_size_int)); + TRY_RESULT(symbols_count, td::narrow_cast_safe(symbols_count_int)); + + if (symbol_size == 0) { + return td::Status::Error("invalid fec type: symbol_size is 0"); + } + if (symbol_size > 1 << 11) { + return td::Status::Error("invalid fec type: symbol_size is too big"); + } + if (symbols_count != (data_size + symbol_size - 1) / symbol_size) { + return td::Status::Error("invalid fec type: wrong symbols_count"); + } FecType T; - ton_api::downcast_call( - *obj.get(), td::overloaded( - [&](const ton_api::fec_raptorQ &obj) { - T.type_ = td::fec::RaptorQEncoder::Parameters{static_cast(obj.data_size_), - static_cast(obj.symbol_size_), - static_cast(obj.symbols_count_)}; - }, - [&](const ton_api::fec_roundRobin &obj) { - T.type_ = td::fec::RoundRobinEncoder::Parameters{static_cast(obj.data_size_), - static_cast(obj.symbol_size_), - static_cast(obj.symbols_count_)}; - }, - [&](const ton_api::fec_online &obj) { - T.type_ = td::fec::OnlineEncoder::Parameters{static_cast(obj.data_size_), - static_cast(obj.symbol_size_), - static_cast(obj.symbols_count_)}; - })); + ton_api::downcast_call(*obj, + td::overloaded( + [&](const ton_api::fec_raptorQ &obj) { + T.type_ = td::fec::RaptorQEncoder::Parameters{data_size, symbol_size, symbols_count}; + }, + [&](const ton_api::fec_roundRobin &obj) { + T.type_ = td::fec::RoundRobinEncoder::Parameters{data_size, symbol_size, symbols_count}; + }, + [&](const ton_api::fec_online &obj) { + T.type_ = td::fec::OnlineEncoder::Parameters{data_size, symbol_size, symbols_count}; + })); return T; } diff --git a/http/http-connection.cpp b/http/http-connection.cpp index 6ba1a813..f1c3c58c 100644 --- a/http/http-connection.cpp +++ b/http/http-connection.cpp @@ -58,9 +58,7 @@ void HttpConnection::loop() { } TRY_STATUS(buffered_fd_.flush_write()); if (writing_payload_ && buffered_fd_.left_unwritten() < fd_high_watermark()) { - auto w = buffered_fd_.left_unwritten(); - continue_payload_write(); - written = buffered_fd_.left_unwritten() > w; + written = continue_payload_write(); } if (close_after_write_ && !writing_payload_ && !buffered_fd_.left_unwritten()) { LOG(INFO) << "close after write"; @@ -120,7 +118,7 @@ void HttpConnection::write_payload(std::shared_ptr payload) { class Cb : public HttpPayload::Callback { public: - Cb(td::actor::ActorId conn) : conn_(conn) { + Cb(td::actor::ActorId conn, size_t watermark) : conn_(conn), watermark_(watermark) { } void run(size_t ready_bytes) override { if (!reached_ && ready_bytes >= watermark_) { @@ -135,23 +133,23 @@ void HttpConnection::write_payload(std::shared_ptr payload) { } private: - size_t watermark_ = chunk_size(); - bool reached_ = false; - td::actor::ActorId conn_; + size_t watermark_; + bool reached_ = false; }; - writing_payload_->add_callback(std::make_unique(actor_id(this))); + writing_payload_->add_callback(std::make_unique( + actor_id(this), writing_payload_->payload_type() == HttpPayload::PayloadType::pt_tunnel ? 1 : chunk_size())); continue_payload_write(); } -void HttpConnection::continue_payload_write() { +bool HttpConnection::continue_payload_write() { if (!writing_payload_) { - return; + return false; } if (writing_payload_->is_error()) { stop(); - return; + return false; } auto t = writing_payload_->payload_type(); @@ -159,19 +157,24 @@ void HttpConnection::continue_payload_write() { t = HttpPayload::PayloadType::pt_chunked; } + bool wrote = false; while (!writing_payload_->written()) { if (buffered_fd_.left_unwritten() > fd_high_watermark()) { - return; + return wrote; } - if (!writing_payload_->parse_completed() && writing_payload_->ready_bytes() < chunk_size()) { - return; + bool is_tunnel = writing_payload_->payload_type() == HttpPayload::PayloadType::pt_tunnel; + if (!is_tunnel && !writing_payload_->parse_completed() && writing_payload_->ready_bytes() < chunk_size()) { + return wrote; } - writing_payload_->store_http(buffered_fd_.output_buffer(), chunk_size(), t); + if (is_tunnel && writing_payload_->ready_bytes() == 0) { + return wrote; + } + wrote |= writing_payload_->store_http(buffered_fd_.output_buffer(), chunk_size(), t); } if (writing_payload_->parse_completed() && writing_payload_->written()) { payload_written(); - return; } + return wrote; } td::Status HttpConnection::read_payload(HttpResponse *response) { diff --git a/http/http-connection.h b/http/http-connection.h index f31af484..840ac8a1 100644 --- a/http/http-connection.h +++ b/http/http-connection.h @@ -65,9 +65,7 @@ class HttpConnection : public td::actor::Actor, public td::ObserverBase { void send_request(std::unique_ptr request, std::shared_ptr payload); void send_response(std::unique_ptr response, std::shared_ptr payload); void write_payload(std::shared_ptr payload); - void continue_payload_write(); - td::Status receive_request(); - td::Status receive_response(); + bool continue_payload_write(); td::Status read_payload(HttpRequest *request); td::Status read_payload(HttpResponse *response); td::Status read_payload(std::shared_ptr payload); diff --git a/http/http-inbound-connection.cpp b/http/http-inbound-connection.cpp index ef2f3a58..89c9c123 100644 --- a/http/http-inbound-connection.cpp +++ b/http/http-inbound-connection.cpp @@ -44,13 +44,22 @@ void HttpInboundConnection::send_server_error() { loop(); } -void HttpInboundConnection::send_proxy_error() { - static const auto s = - "HTTP/1.1 502 Bad Gateway\r\n" - "Connection: keep-alive\r\n" - "Content-length: 0\r\n" - "\r\n"; - buffered_fd_.output_buffer().append(td::Slice(s, strlen(s))); +void HttpInboundConnection::send_proxy_error(td::Status error) { + if (error.code() == ErrorCode::timeout) { + static const auto s = + "HTTP/1.1 504 Gateway Timeout\r\n" + "Connection: keep-alive\r\n" + "Content-length: 0\r\n" + "\r\n"; + buffered_fd_.output_buffer().append(td::Slice(s, strlen(s))); + } else { + static const auto s = + "HTTP/1.1 502 Bad Gateway\r\n" + "Connection: keep-alive\r\n" + "Content-length: 0\r\n" + "\r\n"; + buffered_fd_.output_buffer().append(td::Slice(s, strlen(s))); + } loop(); } @@ -83,7 +92,7 @@ td::Status HttpInboundConnection::receive(td::ChainBufferReader &input) { auto a = R.move_as_ok(); td::actor::send_closure(SelfId, &HttpInboundConnection::send_answer, std::move(a.first), std::move(a.second)); } else { - td::actor::send_closure(SelfId, &HttpInboundConnection::send_proxy_error); + td::actor::send_closure(SelfId, &HttpInboundConnection::send_proxy_error, R.move_as_error()); } }); http_callback_->receive_request(std::move(cur_request_), payload, std::move(P)); diff --git a/http/http-inbound-connection.h b/http/http-inbound-connection.h index 876c3902..e3602267 100644 --- a/http/http-inbound-connection.h +++ b/http/http-inbound-connection.h @@ -35,7 +35,8 @@ class HttpInboundConnection : public HttpConnection { td::Status receive_eof() override { found_eof_ = true; if (reading_payload_) { - if (reading_payload_->payload_type() != HttpPayload::PayloadType::pt_eof) { + if (reading_payload_->payload_type() != HttpPayload::PayloadType::pt_eof && + reading_payload_->payload_type() != HttpPayload::PayloadType::pt_tunnel) { return td::Status::Error("unexpected EOF"); } else { reading_payload_->complete_parse(); @@ -53,7 +54,7 @@ class HttpInboundConnection : public HttpConnection { void send_client_error(); void send_server_error(); - void send_proxy_error(); + void send_proxy_error(td::Status error); void payload_written() override { writing_payload_ = nullptr; diff --git a/http/http-outbound-connection.h b/http/http-outbound-connection.h index a3f5d513..a3d75258 100644 --- a/http/http-outbound-connection.h +++ b/http/http-outbound-connection.h @@ -44,7 +44,8 @@ class HttpOutboundConnection : public HttpConnection { td::Status receive_eof() override { found_eof_ = true; if (reading_payload_) { - if (reading_payload_->payload_type() != HttpPayload::PayloadType::pt_eof) { + if (reading_payload_->payload_type() != HttpPayload::PayloadType::pt_eof && + reading_payload_->payload_type() != HttpPayload::PayloadType::pt_tunnel) { return td::Status::Error("unexpected EOF"); } else { LOG(INFO) << "stopping (EOF payload)"; diff --git a/http/http.cpp b/http/http.cpp index b3c56c9e..cefe1a47 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -164,6 +164,8 @@ td::Result> HttpRequest::create_empty_payload() { if (!need_payload()) { return std::make_shared(HttpPayload::PayloadType::pt_empty); + } else if (method_ == "CONNECT") { + return std::make_shared(HttpPayload::PayloadType::pt_tunnel, low_watermark(), high_watermark()); } else if (found_content_length_) { return std::make_shared(HttpPayload::PayloadType::pt_content_length, low_watermark(), high_watermark(), content_length_); @@ -175,7 +177,7 @@ td::Result> HttpRequest::create_empty_payload() { } bool HttpRequest::need_payload() const { - return found_content_length_ || found_transfer_encoding_; + return found_content_length_ || found_transfer_encoding_ || method_ == "CONNECT"; } td::Status HttpRequest::add_header(HttpHeader header) { @@ -191,9 +193,6 @@ td::Status HttpRequest::add_header(HttpHeader header) { if (found_transfer_encoding_ || found_content_length_) { return td::Status::Error("duplicate Content-Length/Transfer-Encoding"); } - if (len > HttpRequest::max_payload_size()) { - return td::Status::Error("too big Content-Length"); - } content_length_ = len; found_content_length_ = true; } else if (lc_name == "transfer-encoding") { @@ -284,7 +283,8 @@ td::Status HttpPayload::parse(td::ChainBufferReader &input) { case PayloadType::pt_empty: UNREACHABLE(); case PayloadType::pt_eof: - cur_chunk_size_ = 1 << 30; + case PayloadType::pt_tunnel: + cur_chunk_size_ = 1LL << 60; break; case PayloadType::pt_chunked: state_ = ParseState::reading_crlf; @@ -480,17 +480,18 @@ void HttpPayload::run_callbacks() { } } -void HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, HttpPayload::PayloadType store_type) { +bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, HttpPayload::PayloadType store_type) { if (store_type == PayloadType::pt_empty) { - return; + return false; } slice_gc(); + bool wrote = false; while (chunks_.size() > 0 && max_size > 0) { auto cur_state = state_.load(std::memory_order_consume); auto s = get_slice(max_size); if (s.size() == 0) { if (cur_state != ParseState::reading_trailer && cur_state != ParseState::completed) { - return; + return wrote; } else { break; } @@ -500,28 +501,33 @@ void HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt if (store_type == PayloadType::pt_chunked) { char buf[64]; ::sprintf(buf, "%lx\r\n", s.size()); - output.append(td::Slice(buf, strlen(buf))); + auto slice = td::Slice(buf, strlen(buf)); + wrote |= !slice.empty(); + output.append(slice); } + wrote |= !s.empty(); output.append(std::move(s)); if (store_type == PayloadType::pt_chunked) { output.append(td::Slice("\r\n", 2)); + wrote = true; } } - if (chunks_.size() != 0) { - return; + if (chunks_.size() != 0 || !parse_completed()) { + return wrote; } if (!written_zero_chunk_) { if (store_type == PayloadType::pt_chunked) { output.append(td::Slice("0\r\n", 3)); + wrote = true; } written_zero_chunk_ = true; } if (store_type != PayloadType::pt_chunked) { written_trailer_ = true; - return; + return wrote; } while (max_size > 0) { @@ -529,15 +535,16 @@ void HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt HttpHeader h = get_header(); if (h.empty()) { if (cur_state != ParseState::completed) { - return; + return wrote; } else { break; } } auto s = h.size(); h.store_http(output); + wrote = true; if (max_size <= s) { - return; + return wrote; } max_size -= s; } @@ -545,7 +552,9 @@ void HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt if (!written_trailer_) { output.append(td::Slice("\r\n", 2)); written_trailer_ = true; + wrote = true; } + return wrote; } tl_object_ptr HttpPayload::store_tl(size_t max_size) { @@ -729,17 +738,18 @@ td::Result> HttpResponse::parse(std::unique_ptr> HttpResponse::create(std::string proto_version, td::uint32 code, std::string reason, bool force_no_payload, - bool keep_alive) { + bool keep_alive, bool is_tunnel) { if (proto_version != "HTTP/1.0" && proto_version != "HTTP/1.1") { return td::Status::Error(PSTRING() << "unsupported http version '" << proto_version << "'"); } @@ -749,7 +759,7 @@ td::Result> HttpResponse::create(std::string proto } return std::make_unique(std::move(proto_version), code, std::move(reason), force_no_payload, - keep_alive); + keep_alive, is_tunnel); } td::Status HttpResponse::complete_parse_header() { @@ -767,6 +777,8 @@ td::Result> HttpResponse::create_empty_payload() { if (!need_payload()) { return std::make_shared(HttpPayload::PayloadType::pt_empty); + } else if (is_tunnel_) { + return std::make_shared(HttpPayload::PayloadType::pt_tunnel, low_watermark(), high_watermark()); } else if (found_content_length_) { return std::make_shared(HttpPayload::PayloadType::pt_content_length, low_watermark(), high_watermark(), content_length_); @@ -794,9 +806,6 @@ td::Status HttpResponse::add_header(HttpHeader header) { if (found_transfer_encoding_ || found_content_length_) { return td::Status::Error("duplicate Content-Length/Transfer-Encoding"); } - if (len > HttpRequest::max_payload_size()) { - return td::Status::Error("too big Content-Length"); - } content_length_ = len; found_content_length_ = true; } else if (lc_name == "transfer-encoding") { @@ -828,10 +837,12 @@ void HttpResponse::store_http(td::ChainBufferWriter &output) { for (auto &x : options_) { x.store_http(output); } - if (keep_alive_) { - HttpHeader{"Connection", "Keep-Alive"}.store_http(output); - } else { - HttpHeader{"Connection", "Close"}.store_http(output); + if (!is_tunnel_) { + if (keep_alive_) { + HttpHeader{"Connection", "Keep-Alive"}.store_http(output); + } else { + HttpHeader{"Connection", "Close"}.store_http(output); + } } output.append(td::Slice("\r\n", 2)); } @@ -847,7 +858,7 @@ tl_object_ptr HttpResponse::store_tl() { } else { headers.push_back(HttpHeader{"Connection", "Close"}.store_tl()); } - return create_tl_object(proto_version_, code_, reason_, std::move(headers)); + return create_tl_object(proto_version_, code_, reason_, std::move(headers), false); } td::Status HttpHeader::basic_check() { @@ -893,7 +904,9 @@ void answer_error(HttpStatusCode code, std::string reason, } auto response = HttpResponse::create("HTTP/1.0", code, reason, false, false).move_as_ok(); response->add_header(HttpHeader{"Content-Length", "0"}); + response->complete_parse_header(); auto payload = response->create_empty_payload().move_as_ok(); + payload->complete_parse(); CHECK(payload->parse_completed()); promise.set_value(std::make_pair(std::move(response), std::move(payload))); } diff --git a/http/http.h b/http/http.h index f5e2087a..8e109455 100644 --- a/http/http.h +++ b/http/http.h @@ -64,7 +64,7 @@ td::Result get_header(std::string line); class HttpPayload { public: - enum class PayloadType { pt_empty, pt_eof, pt_chunked, pt_content_length }; + enum class PayloadType { pt_empty, pt_eof, pt_chunked, pt_content_length, pt_tunnel }; HttpPayload(PayloadType t, size_t low_watermark, size_t high_watermark, td::uint64 size) : type_(t), low_watermark_(low_watermark), high_watermark_(high_watermark), cur_chunk_size_(size) { CHECK(t == PayloadType::pt_content_length); @@ -75,17 +75,15 @@ class HttpPayload { CHECK(t != PayloadType::pt_content_length); CHECK(t != PayloadType::pt_empty); switch (t) { - case PayloadType::pt_empty: - UNREACHABLE(); case PayloadType::pt_eof: + case PayloadType::pt_tunnel: state_ = ParseState::reading_chunk_data; break; case PayloadType::pt_chunked: state_ = ParseState::reading_chunk_header; break; - case PayloadType::pt_content_length: - state_ = ParseState::reading_chunk_data; - break; + default: + UNREACHABLE(); } } HttpPayload(PayloadType t) : type_(t) { @@ -136,7 +134,7 @@ class HttpPayload { void slice_gc(); HttpHeader get_header(); - void store_http(td::ChainBufferWriter &output, size_t max_size, HttpPayload::PayloadType store_type); + bool store_http(td::ChainBufferWriter &output, size_t max_size, HttpPayload::PayloadType store_type); tl_object_ptr store_tl(size_t max_size); bool written() const { @@ -267,9 +265,11 @@ class HttpResponse { } static td::Result> create(std::string proto_version, td::uint32 code, - std::string reason, bool force_no_payload, bool keep_alive); + std::string reason, bool force_no_payload, bool keep_alive, + bool is_tunnel = false); - HttpResponse(std::string proto_version, td::uint32 code, std::string reason, bool force_no_payload, bool keep_alive); + HttpResponse(std::string proto_version, td::uint32 code, std::string reason, bool force_no_payload, bool keep_alive, + bool is_tunnel = false); bool check_parse_header_completed() const; bool keep_alive() const { @@ -323,6 +323,7 @@ class HttpResponse { bool keep_alive_ = false; std::vector options_; + bool is_tunnel_ = false; }; void answer_error(HttpStatusCode code, std::string reason, diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 2b559bc7..1a08841c 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -1979,10 +1979,11 @@ void TestNode::dns_resolve_finish(ton::WorkchainId workchain, ton::StdSmcAddress if (!dict.check_for_each([this, &out](Ref cs, td::ConstBitPtr key, int n) { CHECK(n == 256); td::Bits256 x{key}; - /*if (cs.is_null() || cs->size_ext() != 0x10000) { + if (cs.is_null() || cs->size_ext() != 0x10000) { out << "category " << x << " : value is not a reference" << std::endl; return true; - }*/ + } + cs = vm::load_cell_slice_ref(cs->prefetch_ref()); std::ostringstream os; (void)show_dns_record(os, x, cs, true); out << "category " << x << " : " << os.str() << std::endl; diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index 4297ea0c..e38ff78d 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -28,8 +28,8 @@ td::Result> BroadcastFec::create(Overlay::Broadcas Overlay::BroadcastDataHash data_hash, td::uint32 flags, td::uint32 date, fec::FecType fec_type, bool is_ours) { auto F = std::make_unique(hash, std::move(src), data_hash, flags, date, std::move(fec_type), is_ours); - TRY_STATUS(F->init_fec_type()); TRY_STATUS(F->run_checks()); + TRY_STATUS(F->init_fec_type()); return std::move(F); } diff --git a/rldp-http-proxy/CMakeLists.txt b/rldp-http-proxy/CMakeLists.txt index 8fa4ac55..7ba66ccb 100644 --- a/rldp-http-proxy/CMakeLists.txt +++ b/rldp-http-proxy/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) -add_executable(rldp-http-proxy rldp-http-proxy.cpp) +add_executable(rldp-http-proxy rldp-http-proxy.cpp DNSResolver.h TonlibClient.h TonlibClient.cpp DNSResolver.cpp) target_include_directories(rldp-http-proxy PUBLIC $) target_link_libraries(rldp-http-proxy PRIVATE tonhttp rldp dht tonlib git) diff --git a/rldp-http-proxy/DNSResolver.cpp b/rldp-http-proxy/DNSResolver.cpp new file mode 100644 index 00000000..a7519742 --- /dev/null +++ b/rldp-http-proxy/DNSResolver.cpp @@ -0,0 +1,105 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "DNSResolver.h" +#include "td/utils/overloaded.h" + +static const double CACHE_TIMEOUT_HARD = 300.0; +static const double CACHE_TIMEOUT_SOFT = 270.0; + +DNSResolver::DNSResolver(td::actor::ActorId tonlib_client) : tonlib_client_(std::move(tonlib_client)) { +} + +void DNSResolver::start_up() { + auto obj = tonlib_api::make_object(); + auto P = td::PromiseCreator::lambda([](td::Result>) {}); + td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P)); +} + +void DNSResolver::resolve(std::string host, td::Promise promise) { + auto it = cache_.find(host); + if (it != cache_.end()) { + const CacheEntry &entry = it->second; + double now = td::Time::now(); + if (now < entry.created_at_ + CACHE_TIMEOUT_HARD) { + promise.set_result(entry.id_); + promise.reset(); + if (now < entry.created_at_ + CACHE_TIMEOUT_SOFT) { + return; + } + } + } + + td::Bits256 category = td::sha256_bits256(td::Slice("site", 4)); + auto obj = tonlib_api::make_object(nullptr, host, category, 16); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise), host = std::move(host)]( + td::Result> R) mutable { + if (R.is_error()) { + if (promise) { + promise.set_result(R.move_as_error()); + } + } else { + auto v = R.move_as_ok(); + auto obj = dynamic_cast(v.get()); + if (obj == nullptr) { + promise.set_result(td::Status::Error("invalid response from tonlib")); + return; + } + ton::adnl::AdnlNodeIdShort id; + td::uint32 cnt = 0; + for (auto &e : obj->entries_) { + tonlib_api::downcast_call(*e->entry_.get(), + td::overloaded( + [&](tonlib_api::dns_entryDataAdnlAddress &x) { + if (td::Random::fast(0, cnt) == 0) { + auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_); + if (R.is_ok()) { + id = R.move_as_ok(); + cnt++; + } + } + }, + [&](auto &x) {})); + } + if (cnt == 0) { + if (promise) { + promise.set_error(td::Status::Error("no DNS entries")); + } + } else { + td::actor::send_closure(SelfId, &DNSResolver::save_to_cache, std::move(host), id); + if (promise) { + promise.set_result(id); + } + } + } + }); + td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P)); +} + +void DNSResolver::save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id) { + CacheEntry &entry = cache_[host]; + entry.id_ = id; + entry.created_at_ = td::Time::now(); +} diff --git a/rldp-http-proxy/DNSResolver.h b/rldp-http-proxy/DNSResolver.h new file mode 100644 index 00000000..f87b5e5a --- /dev/null +++ b/rldp-http-proxy/DNSResolver.h @@ -0,0 +1,49 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#pragma once +#include "td/actor/actor.h" +#include "TonlibClient.h" +#include "adnl/adnl.h" +#include "td/actor/PromiseFuture.h" + +class DNSResolver : public td::actor::Actor { + public: + explicit DNSResolver(td::actor::ActorId tonlib_client); + + void start_up() override; + void resolve(std::string host, td::Promise promise); + + private: + void save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id); + + td::actor::ActorId tonlib_client_; + + struct CacheEntry { + ton::adnl::AdnlNodeIdShort id_; + double created_at_; + }; + std::map cache_; +}; \ No newline at end of file diff --git a/rldp-http-proxy/TonlibClient.cpp b/rldp-http-proxy/TonlibClient.cpp new file mode 100644 index 00000000..a8e18b81 --- /dev/null +++ b/rldp-http-proxy/TonlibClient.cpp @@ -0,0 +1,72 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "TonlibClient.h" + +TonlibClient::TonlibClient(ton::tl_object_ptr options) : options_(std::move(options)) { +} + +void TonlibClient::start_up() { + class Cb : public tonlib::TonlibCallback { + public: + explicit Cb(td::actor::ActorId self_id) : self_id_(self_id) { + } + void on_result(std::uint64_t id, tonlib_api::object_ptr result) override { + td::actor::send_closure(self_id_, &TonlibClient::receive_request_result, id, std::move(result)); + } + void on_error(std::uint64_t id, tonlib_api::object_ptr error) override { + td::actor::send_closure(self_id_, &TonlibClient::receive_request_result, id, + td::Status::Error(error->code_, std::move(error->message_))); + } + + private: + td::actor::ActorId self_id_; + }; + + tonlib_client_ = td::actor::create_actor("tonlibclient", td::make_unique(actor_id(this))); + auto init = tonlib_api::make_object(std::move(options_)); + auto P = td::PromiseCreator::lambda([](td::Result> R) mutable { + R.ensure(); + }); + send_request(std::move(init), std::move(P)); +} + +void TonlibClient::send_request(tonlib_api::object_ptr obj, + td::Promise> promise) { + auto id = next_request_id_++; + CHECK(requests_.emplace(id, std::move(promise)).second); + td::actor::send_closure(tonlib_client_, &tonlib::TonlibClient::request, id, std::move(obj)); +} + +void TonlibClient::receive_request_result(td::uint64 id, td::Result> R) { + if (id == 0) { + return; + } + auto it = requests_.find(id); + CHECK(it != requests_.end()); + auto promise = std::move(it->second); + requests_.erase(it); + promise.set_result(std::move(R)); +} \ No newline at end of file diff --git a/rldp-http-proxy/TonlibClient.h b/rldp-http-proxy/TonlibClient.h new file mode 100644 index 00000000..0b75c6c0 --- /dev/null +++ b/rldp-http-proxy/TonlibClient.h @@ -0,0 +1,47 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#pragma once +#include "td/actor/actor.h" +#include "auto/tl/tonlib_api.hpp" +#include "tonlib/tonlib/TonlibClient.h" + +class TonlibClient : public td::actor::Actor { + public: + explicit TonlibClient(ton::tl_object_ptr options); + + void start_up() override; + + void send_request(tonlib_api::object_ptr obj, + td::Promise> promise); + + private: + void receive_request_result(td::uint64 id, td::Result> R); + + ton::tl_object_ptr options_; + td::actor::ActorOwn tonlib_client_; + std::map>> requests_; + td::uint64 next_request_id_{1}; +}; diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index fb5ae4c1..a5faa002 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -52,6 +52,11 @@ #include #include #include "git.h" +#include "td/utils/BufferedFd.h" +#include "common/delay.h" + +#include "TonlibClient.h" +#include "DNSResolver.h" #if TD_DARWIN || TD_LINUX #include @@ -120,16 +125,28 @@ class HttpRemote : public td::actor::Actor { private: td::IPAddress addr_; - bool ready_ = false; + bool ready_ = true; td::actor::ActorOwn client_; }; +td::BufferSlice create_error_response(const std::string &proto_version, int code, const std::string &reason) { + return ton::create_serialize_tl_object( + proto_version, code, reason, std::vector>(), true); +} + class HttpRldpPayloadReceiver : public td::actor::Actor { public: HttpRldpPayloadReceiver(std::shared_ptr payload, td::Bits256 transfer_id, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort local_id, - td::actor::ActorId adnl, td::actor::ActorId rldp) - : payload_(std::move(payload)), id_(transfer_id), src_(src), local_id_(local_id), adnl_(adnl), rldp_(rldp) { + td::actor::ActorId adnl, td::actor::ActorId rldp, + bool is_tunnel = false) + : payload_(std::move(payload)) + , id_(transfer_id) + , src_(src) + , local_id_(local_id) + , adnl_(adnl) + , rldp_(rldp) + , is_tunnel_(is_tunnel) { } void start_up() override { @@ -178,13 +195,14 @@ class HttpRldpPayloadReceiver : public td::actor::Actor { auto f = ton::create_serialize_tl_object( id_, seqno_++, static_cast(chunk_size())); + auto timeout = td::Timestamp::in(is_tunnel_ ? 60.0 : 15.0); td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, src_, "payload part", std::move(P), - td::Timestamp::in(15.0), std::move(f), 2 * chunk_size() + 1024); + timeout, std::move(f), 2 * chunk_size() + 1024); } void add_data(td::BufferSlice data) { LOG(INFO) << "HttpPayloadReceiver: received answer (size " << data.size() << ")"; - auto F = ton::fetch_tl_object(std::move(data), true); + auto F = ton::fetch_tl_object(data, true); if (F.is_error()) { abort_query(F.move_as_error()); return; @@ -243,14 +261,20 @@ class HttpRldpPayloadReceiver : public td::actor::Actor { bool sent_ = false; td::int32 seqno_ = 0; + bool is_tunnel_; }; class HttpRldpPayloadSender : public td::actor::Actor { public: HttpRldpPayloadSender(std::shared_ptr payload, td::Bits256 transfer_id, ton::adnl::AdnlNodeIdShort local_id, td::actor::ActorId adnl, - td::actor::ActorId rldp) - : payload_(std::move(payload)), id_(transfer_id), local_id_(local_id), adnl_(adnl), rldp_(rldp) { + td::actor::ActorId rldp, bool is_tunnel = false) + : payload_(std::move(payload)) + , id_(transfer_id) + , local_id_(local_id) + , adnl_(adnl) + , rldp_(rldp) + , is_tunnel_(is_tunnel) { } std::string generate_prefix() const { @@ -287,32 +311,36 @@ class HttpRldpPayloadSender : public td::actor::Actor { class Cb : public ton::http::HttpPayload::Callback { public: - Cb(td::actor::ActorId id) : self_id_(id) { + Cb(td::actor::ActorId id, size_t watermark) : self_id_(id), watermark_(watermark) { } void run(size_t ready_bytes) override { if (!reached_ && ready_bytes >= watermark_) { reached_ = true; - td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query); + td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query, false); } else if (reached_ && ready_bytes < watermark_) { reached_ = false; } } void completed() override { - td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query); + td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query, false); } private: - size_t watermark_ = ton::http::HttpRequest::low_watermark(); bool reached_ = false; td::actor::ActorId self_id_; + size_t watermark_; }; - payload_->add_callback(std::make_unique(actor_id(this))); + payload_->add_callback( + std::make_unique(actor_id(this), is_tunnel_ ? 1 : ton::http::HttpRequest::low_watermark())); - alarm_timestamp() = td::Timestamp::in(10.0); + alarm_timestamp() = td::Timestamp::in(is_tunnel_ ? 60.0 : 10.0); } - void try_answer_query() { + void try_answer_query(bool from_timer = false) { + if (from_timer) { + active_timer_ = false; + } if (!cur_query_promise_) { return; } @@ -321,6 +349,17 @@ class HttpRldpPayloadSender : public td::actor::Actor { } if (payload_->parse_completed() || payload_->ready_bytes() >= ton::http::HttpRequest::low_watermark()) { answer_query(); + } else if (!is_tunnel_ || payload_->ready_bytes() == 0) { + return; + } else if (from_timer) { + answer_query(); + } else if (!active_timer_) { + active_timer_ = true; + ton::delay_action( + [SelfId = actor_id(this)]() { + td::actor::send_closure(SelfId, &HttpRldpPayloadSender::try_answer_query, true); + }, + td::Timestamp::in(0.001)); } } @@ -348,16 +387,12 @@ class HttpRldpPayloadSender : public td::actor::Actor { LOG(INFO) << "received request. size=" << cur_query_size_ << " parse_completed=" << payload_->parse_completed() << " ready_bytes=" << payload_->ready_bytes(); - if (payload_->parse_completed() || payload_->ready_bytes() >= ton::http::HttpRequest::low_watermark()) { - answer_query(); - return; - } - - alarm_timestamp() = td::Timestamp::in(10.0); + alarm_timestamp() = td::Timestamp::in(is_tunnel_ ? 50.0 : 10.0); + try_answer_query(false); } void receive_query(td::BufferSlice data, td::Promise promise) { - auto F = ton::fetch_tl_object(std::move(data), true); + auto F = ton::fetch_tl_object(data, true); if (F.is_error()) { LOG(INFO) << "failed to parse query: " << F.move_as_error(); return; @@ -367,6 +402,10 @@ class HttpRldpPayloadSender : public td::actor::Actor { void alarm() override { if (cur_query_promise_) { + if (is_tunnel_) { + answer_query(); + return; + } LOG(INFO) << "timeout on inbound connection. closing http transfer"; } else { LOG(INFO) << "timeout on RLDP connection. closing http transfer"; @@ -382,7 +421,7 @@ class HttpRldpPayloadSender : public td::actor::Actor { } seqno_++; - alarm_timestamp() = td::Timestamp::in(30.0); + alarm_timestamp() = td::Timestamp::in(is_tunnel_ ? 60.0 : 30.0); } void abort_query(td::Status error) { @@ -403,7 +442,6 @@ class HttpRldpPayloadSender : public td::actor::Actor { td::Bits256 id_; - bool sent_ = false; td::int32 seqno_ = 0; ton::adnl::AdnlNodeIdShort local_id_; @@ -412,6 +450,7 @@ class HttpRldpPayloadSender : public td::actor::Actor { size_t cur_query_size_; td::Promise cur_query_promise_; + bool is_tunnel_, active_timer_ = false; }; class RldpHttpProxy; @@ -423,7 +462,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { std::shared_ptr request_payload, td::Promise, std::shared_ptr>> promise, td::actor::ActorId adnl, td::actor::ActorId dht, - td::actor::ActorId rldp, td::actor::ActorId proxy) + td::actor::ActorId rldp, td::actor::ActorId dns_resolver) : local_id_(local_id) , host_(std::move(host)) , request_(std::move(request)) @@ -432,7 +471,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { , adnl_(adnl) , dht_(dht) , rldp_(rldp) - , proxy_(proxy) { + , dns_resolver_(dns_resolver) { } void start_up() override { resolve(); @@ -452,7 +491,8 @@ class TcpToRldpRequestSender : public td::actor::Actor { } }); - td::actor::create_actor("HttpPayloadSender", request_payload_, id_, local_id_, adnl_, rldp_) + td::actor::create_actor("HttpPayloadSender", request_payload_, id_, local_id_, adnl_, rldp_, + is_tunnel()) .release(); auto f = ton::serialize_tl_object(request_->store_tl(id_), true); @@ -461,13 +501,14 @@ class TcpToRldpRequestSender : public td::actor::Actor { } void got_result(td::BufferSlice data) { - auto F = ton::fetch_tl_object(std::move(data), true); + auto F = ton::fetch_tl_object(data, true); if (F.is_error()) { abort_query(F.move_as_error()); return; } auto f = F.move_as_ok(); - auto R = ton::http::HttpResponse::create(f->http_version_, f->status_code_, f->reason_, false, true); + auto R = ton::http::HttpResponse::create(f->http_version_, f->status_code_, f->reason_, f->no_payload_, true, + is_tunnel() && f->status_code_ == 200); if (R.is_error()) { abort_query(R.move_as_error()); return; @@ -497,9 +538,13 @@ class TcpToRldpRequestSender : public td::actor::Actor { td::actor::send_closure(SelfId, &TcpToRldpRequestSender::finished_payload_transfer); } }); - td::actor::create_actor("HttpPayloadReceiver", response_payload_, id_, dst_, local_id_, - adnl_, rldp_) - .release(); + if (f->no_payload_) { + response_payload_->complete_parse(); + } else { + td::actor::create_actor("HttpPayloadReceiver", response_payload_, id_, dst_, local_id_, + adnl_, rldp_, is_tunnel()) + .release(); + } promise_.set_value(std::make_pair(std::move(response_), std::move(response_payload_))); stop(); @@ -511,10 +556,15 @@ class TcpToRldpRequestSender : public td::actor::Actor { void abort_query(td::Status error) { LOG(INFO) << "aborting http over rldp query: " << error; + promise_.set_error(std::move(error)); stop(); } protected: + bool is_tunnel() const { + return request_->method() == "CONNECT"; + } + td::Bits256 id_; ton::adnl::AdnlNodeIdShort local_id_; @@ -529,12 +579,216 @@ class TcpToRldpRequestSender : public td::actor::Actor { td::actor::ActorId adnl_; td::actor::ActorId dht_; td::actor::ActorId rldp_; - td::actor::ActorId proxy_; + td::actor::ActorId dns_resolver_; std::unique_ptr response_; std::shared_ptr response_payload_; }; +class RldpTcpTunnel : public td::actor::Actor, private td::ObserverBase { + public: + RldpTcpTunnel(td::Bits256 transfer_id, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort local_id, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::SocketFd fd) + : id_(transfer_id) + , src_(src) + , local_id_(local_id) + , adnl_(std::move(adnl)) + , rldp_(std::move(rldp)) + , fd_(std::move(fd)) { + } + + void start_up() override { + self_ = actor_id(this); + td::actor::SchedulerContext::get()->get_poll().subscribe(fd_.get_poll_info().extract_pollable_fd(this), + td::PollFlags::ReadWrite()); + + class Cb : public ton::adnl::Adnl::Callback { + public: + explicit Cb(td::actor::ActorId id) : self_id_(std::move(id)) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, + td::BufferSlice data) override { + LOG(INFO) << "rldp tcp tunnel: dropping message"; + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(self_id_, &RldpTcpTunnel::receive_query, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId self_id_; + }; + td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, local_id_, generate_prefix(), + std::make_unique(actor_id(this))); + process(); + } + + void tear_down() override { + LOG(INFO) << "RldpTcpTunnel: tear_down"; + td::actor::send_closure(adnl_, &ton::adnl::Adnl::unsubscribe, local_id_, generate_prefix()); + td::actor::SchedulerContext::get()->get_poll().unsubscribe(fd_.get_poll_info().get_pollable_fd_ref()); + } + + void notify() override { + td::actor::send_closure(self_, &RldpTcpTunnel::process); + } + + void request_data() { + if (close_ || sent_request_) { + return; + } + sent_request_ = true; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &RldpTcpTunnel::got_data_from_rldp, std::move(R)); + }); + + auto f = ton::create_serialize_tl_object(id_, out_seqno_++, 1 << 17); + td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, src_, "payload part", std::move(P), + td::Timestamp::in(60.0), std::move(f), (1 << 18) + 1024); + } + + void receive_query(td::BufferSlice data, td::Promise promise) { + auto F = ton::fetch_tl_object(data, true); + if (F.is_error()) { + LOG(INFO) << "failed to parse query: " << F.error(); + promise.set_error(F.move_as_error()); + return; + } + auto f = F.move_as_ok(); + if (cur_promise_) { + LOG(INFO) << "failed to process query: previous query is active"; + promise.set_error(td::Status::Error("previous query is active")); + return; + } + if (f->seqno_ != cur_seqno_) { + LOG(INFO) << "failed to process query: seqno mismatch"; + promise.set_error(td::Status::Error("seqno mismatch")); + return; + } + LOG(INFO) << "RldpTcpTunnel: received query, seqno=" << cur_seqno_; + cur_promise_ = std::move(promise); + cur_max_chunk_size_ = f->max_chunk_size_; + alarm_timestamp() = td::Timestamp::in(50.0); + process(); + } + + void got_data_from_rldp(td::Result R) { + if (R.is_error()) { + abort(R.move_as_error()); + return; + } + td::BufferSlice data = R.move_as_ok(); + LOG(INFO) << "RldpTcpTunnel: received data from rldp: size=" << data.size(); + sent_request_ = false; + auto F = ton::fetch_tl_object(data, true); + if (F.is_error()) { + abort(F.move_as_error()); + return; + } + auto f = F.move_as_ok(); + fd_.output_buffer().append(std::move(f->data_)); + if (f->last_) { + got_last_part_ = true; + } + process(); + } + + void process() { + if (!close_) { + auto status = [&] { + TRY_STATUS(fd_.flush_read()); + TRY_STATUS(fd_.flush_write()); + close_ = td::can_close(fd_); + return td::Status::OK(); + }(); + if (status.is_error()) { + abort(std::move(status)); + return; + } + } + if (got_last_part_) { + close_ = true; + } + answer_query(); + request_data(); + } + + void answer_query(bool allow_empty = false, bool from_timer = false) { + if (from_timer) { + active_timer_ = false; + } + auto &input = fd_.input_buffer(); + if (cur_promise_ && (!input.empty() || close_ || allow_empty)) { + if (!from_timer && !close_ && !allow_empty && input.size() < ton::http::HttpRequest::low_watermark()) { + if (!active_timer_) { + active_timer_ = true; + ton::delay_action( + [SelfId = actor_id(this)]() { + td::actor::send_closure(SelfId, &RldpTcpTunnel::answer_query, false, true); + }, + td::Timestamp::in(0.001)); + } + return; + } + size_t s = std::min(input.size(), cur_max_chunk_size_); + td::BufferSlice data(s); + LOG(INFO) << "RldpTcpTunnel: sending data to rldp: size=" << data.size(); + input.advance(s, td::as_mutable_slice(data)); + cur_promise_.set_result(ton::create_serialize_tl_object( + std::move(data), std::vector>(), close_)); + ++cur_seqno_; + cur_promise_.reset(); + alarm_timestamp() = td::Timestamp::never(); + if (close_) { + stop(); + return; + } + } + } + + void alarm() override { + answer_query(true, false); + } + + void abort(td::Status status) { + LOG(INFO) << "RldpTcpTunnel error: " << status; + if (cur_promise_) { + cur_promise_.set_error(status.move_as_error()); + } + stop(); + } + + private: + std::string generate_prefix() const { + std::string x(static_cast(36), '\0'); + auto S = td::MutableSlice{x}; + CHECK(S.size() == 36); + + auto id = ton::ton_api::http_getNextPayloadPart::ID; + S.copy_from(td::Slice(reinterpret_cast(&id), 4)); + S.remove_prefix(4); + S.copy_from(id_.as_slice()); + return x; + } + + td::Bits256 id_; + + ton::adnl::AdnlNodeIdShort src_; + ton::adnl::AdnlNodeIdShort local_id_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + td::BufferedFd fd_; + + td::actor::ActorId self_; + + td::int32 cur_seqno_ = 0, cur_max_chunk_size_ = 0; + td::Promise cur_promise_; + td::int32 out_seqno_ = 0; + bool close_ = false, sent_request_ = false, got_last_part_ = false; + bool active_timer_ = false; +}; + class RldpToTcpRequestSender : public td::actor::Actor { public: RldpToTcpRequestSender(td::Bits256 id, ton::adnl::AdnlNodeIdShort local_id, ton::adnl::AdnlNodeIdShort dst, @@ -570,9 +824,11 @@ class RldpToTcpRequestSender : public td::actor::Actor { } void got_result(std::pair, std::shared_ptr> R) { - td::actor::create_actor("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_, - rldp_) - .release(); + if (R.first->need_payload()) { + td::actor::create_actor("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_, + rldp_) + .release(); + } auto f = ton::serialize_tl_object(R.first->store_tl(), true); promise_.set_value(std::move(f)); stop(); @@ -580,7 +836,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { void abort_query(td::Status error) { LOG(INFO) << "aborting http over rldp query: " << error; - promise_.set_error(std::move(error)); + promise_.set_result(create_error_response(request_->proto_version(), 502, "Bad Gateway")); stop(); } @@ -603,8 +859,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { class RldpHttpProxy : public td::actor::Actor { public: - RldpHttpProxy() { - } + RldpHttpProxy() = default; void set_port(td::uint16 port) { if (port_) { @@ -627,29 +882,8 @@ class RldpHttpProxy : public td::actor::Actor { client_port_ = port; } - void set_local_host(std::string name, td::IPAddress remote) { - local_hosts_.emplace_back(std::move(name), std::move(remote)); - } - - void receive_request_result(td::uint64 id, td::Result> R) { - if (id == 0) { - return; - } - auto it = tonlib_requests_.find(id); - CHECK(it != tonlib_requests_.end()); - auto promise = std::move(it->second); - tonlib_requests_.erase(it); - - promise.set_result(std::move(R)); - } - - void send_tonlib_request(tonlib_api::object_ptr obj, - td::Promise> promise) { - auto id = next_tonlib_requests_id_++; - - CHECK(tonlib_requests_.emplace(id, std::move(promise)).second); - - td::actor::send_closure(tonlib_client_, &tonlib::TonlibClient::request, id, std::move(obj)); + void set_local_host(std::string host, td::uint16 port, td::IPAddress remote) { + hosts_[host].ports_[port].remote_addr_ = remote; } td::Status load_global_config() { @@ -666,29 +900,11 @@ class RldpHttpProxy : public td::actor::Actor { TRY_RESULT_PREFIX(dht, ton::dht::Dht::create_global_config(std::move(conf.dht_)), "bad [dht] section: "); dht_config_ = std::move(dht); - class Cb : public tonlib::TonlibCallback { - public: - Cb(td::actor::ActorId self_id) : self_id_(self_id) { - } - void on_result(std::uint64_t id, tonlib_api::object_ptr result) override { - td::actor::send_closure(self_id_, &RldpHttpProxy::receive_request_result, id, std::move(result)); - } - void on_error(std::uint64_t id, tonlib_api::object_ptr error) override { - td::actor::send_closure(self_id_, &RldpHttpProxy::receive_request_result, id, - td::Status::Error(error->code_, std::move(error->message_))); - } - - private: - td::actor::ActorId self_id_; - }; - - tonlib_client_ = td::actor::create_actor("tonlibclient", td::make_unique(actor_id(this))); - return td::Status::OK(); } void store_dht() { - for (auto &serv : local_hosts_) { + for (auto &serv : hosts_) { if (serv.first != "*") { for (auto &serv_id : server_ids_) { ton::PublicKey key = ton::pubkeys::Unenc{"http." + serv.first}; @@ -722,7 +938,7 @@ class RldpHttpProxy : public td::actor::Actor { { auto S = load_global_config(); if (S.is_error()) { - LOG(INFO) << S; + LOG(ERROR) << S; std::_Exit(2); } } @@ -749,23 +965,19 @@ class RldpHttpProxy : public td::actor::Actor { }); td::actor::send_closure(keyring_, &ton::keyring::Keyring::get_public_key, x.pubkey_hash(), std::move(Q)); } - auto Q = td::PromiseCreator::lambda( - [promise = ig.get_promise()](td::Result> R) mutable { - R.ensure(); - promise.set_value(td::Unit()); - }); auto conf_dataR = td::read_file(global_config_); conf_dataR.ensure(); - auto req = tonlib_api::make_object(tonlib_api::make_object( + auto tonlib_options = tonlib_api::make_object( tonlib_api::make_object(conf_dataR.move_as_ok().as_slice().str(), "", false, false), - tonlib_api::make_object())); - send_tonlib_request(std::move(req), std::move(Q)); + tonlib_api::make_object()); + tonlib_client_ = td::actor::create_actor("tonlibclient", std::move(tonlib_options)); + dns_resolver_ = td::actor::create_actor("dnsresolver", tonlib_client_.get()); } void run_cont() { - if (is_client_ && local_hosts_.size() > 0) { + if (is_client_ && hosts_.size() > 0) { LOG(ERROR) << "client-only node cannot be server"; std::_Exit(2); } @@ -876,9 +1088,6 @@ class RldpHttpProxy : public td::actor::Actor { ton::adnl::Adnl::int_to_bytestring(ton::ton_api::http_request::ID), std::make_unique(actor_id(this))); } - for (auto &serv : local_hosts_) { - servers_.emplace(serv.first, td::actor::create_actor("remote", serv.second)); - } rldp_ = ton::rldp::Rldp::create(adnl_.get()); td::actor::send_closure(rldp_, &ton::rldp::Rldp::add_id, local_id_); @@ -931,23 +1140,33 @@ class RldpHttpProxy : public td::actor::Actor { td::actor::create_actor("outboundreq", local_id_, host, std::move(request), std::move(payload), std::move(promise), adnl_.get(), dht_.get(), - rldp_.get(), actor_id(this)) + rldp_.get(), dns_resolver_.get()) .release(); } void receive_rldp_request(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { LOG(INFO) << "got HTTP request over rldp from " << src; - TRY_RESULT_PROMISE(promise, f, ton::fetch_tl_object(std::move(data), true)); - TRY_RESULT_PROMISE(promise, request, ton::http::HttpRequest::create(f->method_, f->url_, f->http_version_)); - for (auto &x : f->headers_) { - ton::http::HttpHeader h{x->name_, x->value_}; - TRY_STATUS_PROMISE(promise, h.basic_check()); - request->add_header(std::move(h)); + TRY_RESULT_PROMISE(promise, f, ton::fetch_tl_object(data, true)); + std::unique_ptr request; + auto S = [&]() { + TRY_RESULT_ASSIGN(request, ton::http::HttpRequest::create(f->method_, f->url_, f->http_version_)); + for (auto &x : f->headers_) { + ton::http::HttpHeader h{x->name_, x->value_}; + TRY_STATUS(h.basic_check()); + request->add_header(std::move(h)); + } + TRY_STATUS(request->complete_parse_header()); + return td::Status::OK(); + }(); + if (S.is_error()) { + LOG(INFO) << "Failed to parse http request: " << S; + promise.set_result(create_error_response(f->http_version_, 400, "Bad Request")); + return; } - TRY_STATUS_PROMISE(promise, request->complete_parse_header()); auto host = request->host(); - if (host.size() == 0) { + td::uint16 port = 80; + if (host.empty()) { host = request->url(); if (host.size() >= 7 && host.substr(0, 7) == "http://") { host = host.substr(7); @@ -972,29 +1191,69 @@ class RldpHttpProxy : public td::actor::Actor { { auto p = host.find(':'); if (p != std::string::npos) { + try { + port = (td::uint16)std::stoul(host.substr(p + 1)); + } catch (const std::logic_error &) { + port = 80; + promise.set_result(create_error_response(f->http_version_, 400, "Bad Request")); + return; + } host = host.substr(0, p); } } std::transform(host.begin(), host.end(), host.begin(), [](unsigned char c) { return std::tolower(c); }); - auto it = servers_.find(host); - if (it == servers_.end()) { - it = servers_.find("*"); - if (it == servers_.end()) { - promise.set_error(td::Status::Error(ton::ErrorCode::error, "unknown server name")); + auto it = hosts_.find(host); + if (it == hosts_.end()) { + it = hosts_.find("*"); + if (it == hosts_.end()) { + promise.set_result(create_error_response(f->http_version_, 502, "Bad Gateway")); return; } } + auto it2 = it->second.ports_.find(port); + if (it2 == it->second.ports_.end()) { + promise.set_result(create_error_response(f->http_version_, 502, "Bad Gateway")); + return; + } + auto &server = it2->second; + if (request->method() == "CONNECT") { + LOG(INFO) << "starting HTTP tunnel over RLDP to " << server.remote_addr_; + start_tcp_tunnel(f->id_, src, dst, f->http_version_, server.remote_addr_, std::move(promise)); + return; + } - TRY_RESULT_PROMISE(promise, payload, request->create_empty_payload()); + if (server.http_remote_.empty()) { + server.http_remote_ = td::actor::create_actor("remote", server.remote_addr_); + } + + auto payload = request->create_empty_payload(); + if (payload.is_error()) { + promise.set_result(create_error_response(f->http_version_, 502, "Bad Gateway")); + return; + } LOG(INFO) << "starting HTTP over RLDP request"; td::actor::create_actor("inboundreq", f->id_, dst, src, std::move(request), - std::move(payload), std::move(promise), adnl_.get(), rldp_.get(), - it->second.get()) + payload.move_as_ok(), std::move(promise), adnl_.get(), rldp_.get(), + server.http_remote_.get()) .release(); } + void start_tcp_tunnel(td::Bits256 id, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort local_id, + std::string http_version, td::IPAddress ip, td::Promise promise) { + auto fd = td::SocketFd::open(ip); + if (fd.is_error()) { + promise.set_result(create_error_response(http_version, 502, "Bad Gateway")); + return; + } + td::actor::create_actor(td::actor::ActorOptions().with_name("tunnel").with_poll(), id, src, local_id, + adnl_.get(), rldp_.get(), fd.move_as_ok()).release(); + promise.set_result(ton::create_serialize_tl_object( + http_version, 200, "Connection Established", std::vector>(), + false)); + } + void add_adnl_addr(ton::adnl::AdnlNodeIdShort id) { server_ids_.insert(id); } @@ -1008,10 +1267,17 @@ class RldpHttpProxy : public td::actor::Actor { } private: + struct Host { + struct Server { + td::IPAddress remote_addr_; + td::actor::ActorOwn http_remote_; + }; + std::map ports_; + }; + td::uint16 port_{0}; td::IPAddress addr_; std::string global_config_; - std::vector> local_hosts_; bool is_client_{false}; td::uint16 client_port_{0}; @@ -1022,8 +1288,7 @@ class RldpHttpProxy : public td::actor::Actor { ton::adnl::AdnlNodeIdShort dht_id_; td::actor::ActorOwn server_; - std::map dns_; - std::map> servers_; + std::map hosts_; td::actor::ActorOwn keyring_; td::actor::ActorOwn adnl_network_manager_; @@ -1036,9 +1301,8 @@ class RldpHttpProxy : public td::actor::Actor { std::string db_root_ = "."; bool proxy_all_ = false; - td::actor::ActorOwn tonlib_client_; - std::map>> tonlib_requests_; - td::uint64 next_tonlib_requests_id_{1}; + td::actor::ActorOwn tonlib_client_; + td::actor::ActorOwn dns_resolver_; }; void TcpToRldpRequestSender::resolve() { @@ -1053,63 +1317,15 @@ void TcpToRldpRequestSender::resolve() { resolved(R.move_as_ok()); return; } - if (false) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, R.move_as_error()); - return; - } - auto value = R.move_as_ok(); - if (value.value().size() != 32) { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, td::Status::Error("bad value in dht")); - return; - } - - ton::PublicKeyHash h{value.value().as_slice()}; - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolved, ton::adnl::AdnlNodeIdShort{h}); - }); - - ton::PublicKey key = ton::pubkeys::Unenc{"http." + host_}; - ton::dht::DhtKey dht_key{key.compute_short_id(), "http." + host_, 0}; - td::actor::send_closure(dht_, &ton::dht::Dht::get_value, std::move(dht_key), std::move(P)); - } else { - td::Bits256 category = td::sha256_bits256(td::Slice("site", 4)); - auto obj = tonlib_api::make_object(nullptr, host_, category, 16); - - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, - R.move_as_error_prefix("failed to resolve: ")); - } else { - auto v = R.move_as_ok(); - auto obj = static_cast(v.get()); - ton::adnl::AdnlNodeIdShort id; - td::uint32 cnt = 0; - for (auto &e : obj->entries_) { - tonlib_api::downcast_call( - *e->entry_.get(), td::overloaded( - [&](tonlib_api::dns_entryDataAdnlAddress &x) { - if (td::Random::fast(0, cnt) == 0) { - auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_); - if (R.is_ok()) { - id = R.move_as_ok(); - cnt++; - } - } - }, - [&](auto &x) {})); - } - if (cnt == 0) { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, - td::Status::Error(ton::ErrorCode::notready, "failed to resolve")); - } else { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolved, id); - } - } - }); - td::actor::send_closure(proxy_, &RldpHttpProxy::send_tonlib_request, std::move(obj), std::move(P)); - } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, + R.move_as_error_prefix("failed to resolve: ")); + } else { + td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolved, R.move_as_ok()); + } + }); + td::actor::send_closure(dns_resolver_, &DNSResolver::resolve, host_, std::move(P)); } int main(int argc, char *argv[]) { @@ -1123,6 +1339,42 @@ int main(int argc, char *argv[]) { td::log_interface = td::default_log_interface; }; + auto add_local_host = [&](const std::string& local, const std::string& remote) -> td::Status { + std::string host; + std::vector ports; + auto p = local.find(':'); + if (p == std::string::npos) { + host = local; + ports = {80, 443}; + } else { + host = local.substr(0, p); + ++p; + while (p < local.size()) { + auto p2 = local.find(',', p); + if (p2 == std::string::npos) { + p2 = local.size(); + } + try { + ports.push_back((td::uint16)std::stoul(local.substr(p, p2 - p))); + } catch (const std::logic_error& e) { + return td::Status::Error(PSLICE() << "Invalid port: " << local.substr(p, p2 - p)); + } + p = p2 + 1; + } + } + for (td::uint16 port : ports) { + std::string cur_remote = remote; + if (cur_remote.find(':') == std::string::npos) { + cur_remote += ':'; + cur_remote += std::to_string(port); + } + td::IPAddress addr; + TRY_STATUS(addr.init_host_port(cur_remote)); + td::actor::send_closure(x, &RldpHttpProxy::set_local_host, host, port, addr); + } + return td::Status::OK(); + }; + td::OptionParser p; p.set_description( "A simple rldp-to-http and http-to-rldp proxy for running and accessing ton sites\n" @@ -1136,7 +1388,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows rldp-http-proxy build information", [&]() { - std::cout << "rldp-http-proxy build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "rldp-http-proxy build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints a help message", [&]() { @@ -1170,27 +1423,25 @@ int main(int argc, char *argv[]) { }); p.add_option('C', "global-config", "global TON configuration file", [&](td::Slice arg) { td::actor::send_closure(x, &RldpHttpProxy::set_global_config, arg.str()); }); - p.add_checked_option('L', "local", "http hostname that will be proxied to http server at localhost:80", + p.add_checked_option('L', "local", + ":, hostname that will be proxied to localhost\n" + " is a comma-separated list of ports (may be omitted, default: 80, 443)\n", [&](td::Slice arg) -> td::Status { - td::IPAddress addr; - TRY_STATUS(addr.init_ipv4_port("127.0.0.1", 80)); - td::actor::send_closure(x, &RldpHttpProxy::set_local_host, arg.str(), addr); - return td::Status::OK(); + return add_local_host(arg.str(), "127.0.0.1"); }); p.add_option('D', "db", "db root", [&](td::Slice arg) { td::actor::send_closure(x, &RldpHttpProxy::set_db_root, arg.str()); }); p.add_checked_option( 'R', "remote", - "@:, indicates a http hostname that will be proxied to remote http server at :", + ":@:, indicates a hostname that will be proxied to remote server at :\n" + " is a comma-separated list of ports (may be omitted, default: 80,433)\n" + " is a remote port (may be omitted, default: same as host's port)", [&](td::Slice arg) -> td::Status { auto ch = arg.find('@'); if (ch == td::Slice::npos) { return td::Status::Error("bad format for --remote"); } - td::IPAddress addr; - TRY_STATUS(addr.init_host_port(arg.substr(ch + 1).str())); - td::actor::send_closure(x, &RldpHttpProxy::set_local_host, arg.substr(0, ch).str(), addr); - return td::Status::OK(); + return add_local_host(arg.substr(0, ch).str(), arg.substr(ch + 1).str()); }); p.add_option('d', "daemonize", "set SIGHUP", [&]() { td::set_signal_handler(td::SignalType::HangUp, [](int sig) { diff --git a/rldp/rldp-peer.cpp b/rldp/rldp-peer.cpp index 6192d070..c599e949 100644 --- a/rldp/rldp-peer.cpp +++ b/rldp/rldp-peer.cpp @@ -125,6 +125,10 @@ void RldpTransferReceiverImpl::receive_part(fec::FecType fec_type, td::uint32 pa } if (!decoder_) { + if (offset_ + fec_type.size() > total_size_) { + VLOG(RLDP_NOTICE) << "failed to create decoder: data size in fec type is too big"; + return; + } auto D = fec_type.create_decoder(); if (D.is_error()) { VLOG(RLDP_WARNING) << "failed to create decoder: " << D.move_as_error(); diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index acd77f63..f33a30dc 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -60,12 +60,12 @@ StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) { return string_builder << format::as_time(timer.elapsed()); } -PerfWarningTimer::PerfWarningTimer(string name, double max_duration) - : name_(std::move(name)), start_at_(Time::now()), max_duration_(max_duration) { +PerfWarningTimer::PerfWarningTimer(string name, double max_duration, std::function&& callback) + : name_(std::move(name)), start_at_(Time::now()), max_duration_(max_duration), callback_(std::move(callback)) { } PerfWarningTimer::PerfWarningTimer(PerfWarningTimer &&other) - : name_(std::move(other.name_)), start_at_(other.start_at_), max_duration_(other.max_duration_) { + : name_(std::move(other.name_)), start_at_(other.start_at_), max_duration_(other.max_duration_), callback_(std::move(other.callback_)) { other.start_at_ = 0; } @@ -78,8 +78,9 @@ void PerfWarningTimer::reset() { return; } double duration = Time::now() - start_at_; - LOG_IF(WARNING, duration > max_duration_) - << "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration)); + //LOG_IF(WARNING, duration > max_duration_) + //<< "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration)); + callback_(duration); start_at_ = 0; } diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 7dec45cf..64f3e934 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -20,6 +20,8 @@ #include "td/utils/StringBuilder.h" +#include + namespace td { class Timer { @@ -44,7 +46,7 @@ class Timer { class PerfWarningTimer { public: - explicit PerfWarningTimer(string name, double max_duration = 0.1); + explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function&& callback = [] (double) {}); PerfWarningTimer(const PerfWarningTimer &) = delete; PerfWarningTimer &operator=(const PerfWarningTimer &) = delete; PerfWarningTimer(PerfWarningTimer &&other); @@ -56,6 +58,7 @@ class PerfWarningTimer { string name_; double start_at_{0}; double max_duration_{0}; + std::function callback_; }; } // namespace td diff --git a/tdutils/td/utils/port/config.h b/tdutils/td/utils/port/config.h index 2bd671b0..77143668 100644 --- a/tdutils/td/utils/port/config.h +++ b/tdutils/td/utils/port/config.h @@ -39,7 +39,7 @@ #define TD_EVENTFD_BSD 1 #elif TD_EMSCRIPTEN #define TD_POLL_POLL 1 - #define TD_EVENTFD_UNSUPPORTED 1 + // #define TD_EVENTFD_UNSUPPORTED 1 #elif TD_DARWIN #define TD_POLL_KQUEUE 1 #define TD_EVENTFD_BSD 1 @@ -51,7 +51,11 @@ #endif #if TD_EMSCRIPTEN - #define TD_THREAD_UNSUPPORTED 1 + // #define TD_THREAD_UNSUPPORTED 1 + #define TD_POLL_EPOLL 1 + #define TD_EVENTFD_UNSUPPORTED 0 + #define TD_THREAD_PTHREAD 1 + #define TD_EVENTFD_LINUX 1 #elif TD_TIZEN || TD_LINUX || TD_DARWIN #define TD_THREAD_PTHREAD 1 #else diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index d31ef569..b2a72dc9 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -52,6 +52,8 @@ liteServer.partialBlockProof complete:Bool from:tonNode.blockIdExt to:tonNode.bl liteServer.configInfo mode:# id:tonNode.blockIdExt state_proof:bytes config_proof:bytes = liteServer.ConfigInfo; liteServer.validatorStats mode:# id:tonNode.blockIdExt count:int complete:Bool state_proof:bytes data_proof:bytes = liteServer.ValidatorStats; liteServer.libraryResult result:(vector liteServer.libraryEntry) = liteServer.LibraryResult; +liteServer.shardBlockLink id:tonNode.blockIdExt proof:bytes = liteServer.ShardBlockLink; +liteServer.shardBlockProof masterchain_id:tonNode.blockIdExt links:(vector liteServer.shardBlockLink) = liteServer.ShardBlockProof; liteServer.debug.verbosity value:int = liteServer.debug.Verbosity; @@ -78,6 +80,7 @@ liteServer.getConfigAll mode:# id:tonNode.blockIdExt = liteServer.ConfigInfo; liteServer.getConfigParams mode:# id:tonNode.blockIdExt param_list:(vector int) = liteServer.ConfigInfo; liteServer.getValidatorStats#091a58bc mode:# id:tonNode.blockIdExt limit:int start_after:mode.0?int256 modified_after:mode.2?int = liteServer.ValidatorStats; liteServer.getLibraries library_list:(vector int256) = liteServer.LibraryResult; +liteServer.getShardBlockProof id:tonNode.blockIdExt = liteServer.ShardBlockProof; liteServer.queryPrefix = Object; liteServer.query data:bytes = Object; diff --git a/tl/generate/scheme/lite_api.tlo b/tl/generate/scheme/lite_api.tlo index eb8e3211..a015d932 100644 Binary files a/tl/generate/scheme/lite_api.tlo and b/tl/generate/scheme/lite_api.tlo differ diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 67b123c8..f04f7863 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -651,6 +651,10 @@ engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; +engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; +engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; +engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByName) = engine.validator.PerfTimerStats; + engine.validator.validatorSessionInfo current_block:tonNode.blockId self:int256 current_round:int next_producers:(vector int256) = engine.validator.ValidatorSessionInfo; engine.validator.validatorSessionsInfo sessions:(vector engine.validator.validatorSessionInfo) = engine.validator.ValidatorSessionsInfo; @@ -705,6 +709,8 @@ engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short sign engine.validator.signShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash expire_at:int max_size:int = overlay.Certificate; engine.validator.importShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; +engine.validator.getPerfTimerStats name:string = engine.validator.PerfTimerStats; + engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo; engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; @@ -735,7 +741,7 @@ storage.queryPrefix id:int256 = Object; http.header name:string value:string = http.Header; http.payloadPart data:bytes trailer:(vector http.header) last:Bool = http.PayloadPart; -http.response http_version:string status_code:int reason:string headers:(vector http.header) = http.Response; +http.response http_version:string status_code:int reason:string headers:(vector http.header) no_payload:Bool = http.Response; ---functions--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 25b64d21..d8940288 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index c4d8eb98..79185d5b 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -217,6 +217,12 @@ blocks.transactions id:ton.blockIdExt req_count:int32 incomplete:Bool transactio blocks.header id:ton.blockIdExt global_id:int32 version:int32 flags:# after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 gen_utime:int53 vert_seqno:# prev_blocks:vector = blocks.Header; //blocks.shortData header:blocks.Header transactions:blocks.Header = blocks.BlockData; +blocks.signature node_id_short:int256 signature:bytes = blocks.Signature; +blocks.blockSignatures id:ton.blockIdExt signatures:(vector blocks.signature) = blocks.BlockSignatures; +blocks.shardBlockLink id:ton.blockIdExt proof:bytes = blocks.ShardBlockLink; +blocks.blockLinkBack to_key_block:Bool from:ton.blockIdExt to:ton.blockIdExt dest_proof:bytes proof:bytes state_proof:bytes = blocks.BlockLinkBack; +blocks.shardBlockProof from:ton.blockIdExt mc_id:ton.blockIdExt links:(vector blocks.shardBlockLink) mc_proof:(vector blocks.blockLinkBack) = blocks.ShardBlockProof; + configInfo config:tvm.cell = ConfigInfo; ---functions--- @@ -251,6 +257,7 @@ getBip39Hints prefix:string = Bip39Hints; //raw.init initial_account_state:raw.initialAccountState = Ok; raw.getAccountState account_address:accountAddress = raw.FullAccountState; raw.getTransactions private_key:InputKey account_address:accountAddress from_transaction_id:internal.transactionId = raw.Transactions; +raw.getTransactionsV2 private_key:InputKey account_address:accountAddress from_transaction_id:internal.transactionId count:# try_decode_messages:Bool = raw.Transactions; raw.sendMessage body:bytes = Ok; raw.sendMessageReturnHash body:bytes = raw.ExtMessageInfo; raw.createAndSendMessage destination:accountAddress initial_account_state:bytes data:bytes = Ok; @@ -306,6 +313,8 @@ blocks.getShards id:ton.blockIdExt = blocks.Shards; blocks.lookupBlock mode:int32 id:ton.blockId lt:int64 utime:int32 = ton.BlockIdExt; blocks.getTransactions id:ton.blockIdExt mode:# count:# after:blocks.accountTransactionId = blocks.Transactions; blocks.getBlockHeader id:ton.blockIdExt = blocks.Header; +blocks.getMasterchainBlockSignatures seqno:int32 = blocks.BlockSignatures; +blocks.getShardBlockProof id:ton.blockIdExt mode:# from:mode.0?ton.blockIdExt = blocks.ShardBlockProof; onLiteServerQueryResult id:int64 bytes:bytes = Ok; onLiteServerQueryError id:int64 error:error = Ok; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 69cf1595..ee457609 100644 Binary files a/tl/generate/scheme/tonlib_api.tlo and b/tl/generate/scheme/tonlib_api.tlo differ diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index 5b6530a6..061778af 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -49,6 +49,8 @@ set(TONLIB_SOURCE set(TONLIB_OFFLINE_TEST_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test/offline.cpp PARENT_SCOPE) set(TONLIB_ONLINE_TEST_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test/online.cpp PARENT_SCOPE) +set(USE_EMSCRIPTEN ${USE_EMSCRIPTEN} PARENT_SCOPE) + add_library(tonlib STATIC ${TONLIB_SOURCE}) target_include_directories(tonlib PUBLIC @@ -87,8 +89,12 @@ set(TONLIB_JSON_HEADERS tonlib/tonlib_client_json.h) set(TONLIB_JSON_SOURCE tonlib/tonlib_client_json.cpp) include(GenerateExportHeader) +if (NOT USE_EMSCRIPTEN) + add_library(tonlibjson SHARED ${TONLIB_JSON_SOURCE} ${TONLIB_JSON_HEADERS}) +else() + add_library(tonlibjson STATIC ${TONLIB_JSON_SOURCE} ${TONLIB_JSON_HEADERS}) +endif() -add_library(tonlibjson SHARED ${TONLIB_JSON_SOURCE} ${TONLIB_JSON_HEADERS}) target_link_libraries(tonlibjson PRIVATE tonlibjson_private) generate_export_header(tonlibjson EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h) target_include_directories(tonlibjson PUBLIC @@ -149,11 +155,13 @@ endif() install(FILES ${TONLIB_JSON_HEADERS} ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h DESTINATION include/tonlib/) -install(EXPORT Tonlib - FILE TonlibTargets.cmake - NAMESPACE Tonlib:: - DESTINATION lib/cmake/Tonlib -) +if (NOT USE_EMSCRIPTEN) + install(EXPORT Tonlib + FILE TonlibTargets.cmake + NAMESPACE Tonlib:: + DESTINATION lib/cmake/Tonlib + ) +endif() include(CMakePackageConfigHelpers) write_basic_package_version_file("TonlibConfigVersion.cmake" VERSION ${TON_VERSION} diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index e8a9ac44..842ea5fd 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -41,6 +41,7 @@ #include "block/check-proof.h" #include "ton/lite-tl.hpp" #include "ton/ton-shard.h" +#include "lite-client/lite-client-common.h" #include "vm/boc.h" #include "vm/cellops.h" @@ -960,11 +961,12 @@ td::Result to_balance(td::Ref balance_ref) { class GetTransactionHistory : public td::actor::Actor { public: - GetTransactionHistory(ExtClientRef ext_client_ref, block::StdAddress address, ton::LogicalTime lt, ton::Bits256 hash, + GetTransactionHistory(ExtClientRef ext_client_ref, block::StdAddress address, ton::LogicalTime lt, ton::Bits256 hash, td::int32 count, td::actor::ActorShared<> parent, td::Promise promise) : address_(std::move(address)) , lt_(std::move(lt)) , hash_(std::move(hash)) + , count_(count) , parent_(std::move(parent)) , promise_(std::move(promise)) { client_.set_client(ext_client_ref); @@ -975,7 +977,7 @@ class GetTransactionHistory : public td::actor::Actor { ton::LogicalTime lt_; ton::Bits256 hash_; ExtClient client_; - td::int32 count_{10}; + td::int32 count_; td::actor::ActorShared<> parent_; td::Promise promise_; @@ -1316,6 +1318,321 @@ class GetRawAccountState : public td::actor::Actor { } }; +class GetMasterchainBlockSignatures : public td::actor::Actor { + public: + GetMasterchainBlockSignatures(ExtClientRef ext_client_ref, ton::BlockSeqno seqno, td::actor::ActorShared<> parent, + td::Promise>&& promise) + : block_id_short_(ton::masterchainId, ton::shardIdAll, seqno) + , parent_(std::move(parent)) + , promise_(std::move(promise)) { + client_.set_client(ext_client_ref); + } + + void start_up() override { + if (block_id_short_.seqno == 0) { + abort(td::Status::Error("can't get signatures of block #0")); + return; + } + client_.with_last_block([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::got_last_block, R.ok().last_block_id); + } + }); + } + + void got_last_block(ton::BlockIdExt id) { + last_block_ = id; + prev_block_id_short_ = block_id_short_; + prev_block_id_short_.seqno--; + client_.send_query( + ton::lite_api::liteServer_lookupBlock(1, ton::create_tl_lite_block_id_simple(prev_block_id_short_), 0, 0), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::got_prev_block_id, + ton::create_block_id(R.ok()->id_)); + } + }); + } + + void got_prev_block_id(ton::BlockIdExt id) { + prev_block_id_ = id; + if (prev_block_id_.id != prev_block_id_short_) { + abort(td::Status::Error("got incorrect block header from liteserver")); + return; + } + client_.send_query( + ton::lite_api::liteServer_getBlockProof(0x1001, ton::create_tl_lite_block_id(last_block_), + ton::create_tl_lite_block_id(prev_block_id_)), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::got_prev_proof, R.move_as_ok()); + } + }); + } + + void got_prev_proof(lite_api_ptr proof) { + auto R = liteclient::deserialize_proof_chain(std::move(proof)); + if (R.is_error()) { + abort(R.move_as_error()); + return; + } + auto chain = R.move_as_ok(); + if (chain->from != last_block_ || chain->to != prev_block_id_ || !chain->complete) { + abort(td::Status::Error("got invalid proof chain")); + return; + } + auto S = chain->validate(); + if (S.is_error()) { + abort(std::move(S)); + return; + } + client_.send_query( + ton::lite_api::liteServer_lookupBlock(1, ton::create_tl_lite_block_id_simple(block_id_short_), 0, 0), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::got_block_id, + ton::create_block_id(R.ok()->id_)); + } + }); + } + + void got_block_id(ton::BlockIdExt id) { + block_id_ = id; + client_.send_query( + ton::lite_api::liteServer_getBlockProof(0x1001, ton::create_tl_lite_block_id(prev_block_id_), + ton::create_tl_lite_block_id(block_id_)), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::got_proof, R.move_as_ok()); + } + }); + } + + void got_proof(lite_api_ptr proof) { + auto R = liteclient::deserialize_proof_chain(std::move(proof)); + if (R.is_error()) { + abort(R.move_as_error()); + return; + } + auto chain = R.move_as_ok(); + if (chain->from != prev_block_id_ || chain->to != block_id_ || !chain->complete || chain->links.empty() || + chain->last_link().signatures.empty()) { + abort(td::Status::Error("got invalid proof chain")); + return; + } + auto S = chain->validate(); + if (S.is_error()) { + abort(std::move(S)); + return; + } + std::vector> signatures; + for (const auto& s : chain->last_link().signatures) { + signatures.push_back(ton::create_tl_object(s.node, s.signature.as_slice().str())); + } + promise_.set_result( + ton::create_tl_object(to_tonlib_api(block_id_), std::move(signatures))); + stop(); + } + + void abort(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + ton::BlockId block_id_short_; + td::actor::ActorShared<> parent_; + td::Promise> promise_; + ExtClient client_; + ton::BlockIdExt block_id_; + ton::BlockId prev_block_id_short_; + ton::BlockIdExt prev_block_id_; + ton::BlockIdExt last_block_; +}; + +class GetShardBlockProof : public td::actor::Actor { + public: + GetShardBlockProof(ExtClientRef ext_client_ref, ton::BlockIdExt id, ton::BlockIdExt from, + td::actor::ActorShared<> parent, + td::Promise>&& promise) + : id_(id), from_(from), parent_(std::move(parent)), promise_(std::move(promise)) { + client_.set_client(ext_client_ref); + } + + void start_up() override { + if (from_.is_masterchain_ext()) { + got_from_block(from_); + } else { + client_.with_last_block([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetShardBlockProof::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetShardBlockProof::got_from_block, R.move_as_ok().last_block_id); + } + }); + } + } + + void got_from_block(ton::BlockIdExt from) { + from_ = from; + CHECK(from_.is_masterchain_ext()); + client_.send_query( + ton::lite_api::liteServer_getShardBlockProof(ton::create_tl_lite_block_id(id_)), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetShardBlockProof::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetShardBlockProof::got_shard_block_proof, R.move_as_ok()); + } + }); + } + + void got_shard_block_proof(lite_api_ptr result) { + mc_id_ = create_block_id(std::move(result->masterchain_id_)); + if (!mc_id_.is_masterchain_ext()) { + abort(td::Status::Error("got invalid masterchain block id")); + return; + } + if (result->links_.size() > 8) { + abort(td::Status::Error("chain is too long")); + return; + } + ton::BlockIdExt cur_id = mc_id_; + try { + for (auto& link : result->links_) { + ton::BlockIdExt prev_id = create_block_id(link->id_); + td::BufferSlice proof = std::move(link->proof_); + auto R = vm::std_boc_deserialize(proof); + if (R.is_error()) { + abort(TonlibError::InvalidBagOfCells("proof")); + return; + } + auto block_root = vm::MerkleProof::virtualize(R.move_as_ok(), 1); + if (cur_id.root_hash != block_root->get_hash().bits()) { + abort(td::Status::Error("invalid block hash in proof")); + return; + } + if (cur_id.is_masterchain()) { + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + if (!tlb::unpack_cell(block_root, blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() || + !tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) { + abort(td::Status::Error("cannot unpack block header")); + return; + } + block::ShardConfig shards(mc_extra.shard_hashes->prefetch_ref()); + td::Ref shard_hash = shards.get_shard_hash(prev_id.shard_full(), true); + if (shard_hash.is_null() || shard_hash->top_block_id() != prev_id) { + abort(td::Status::Error("invalid proof chain: prev block is not in mc shard list")); + return; + } + } else { + std::vector prev; + ton::BlockIdExt mc_blkid; + bool after_split; + td::Status S = block::unpack_block_prev_blk_try(block_root, cur_id, prev, mc_blkid, after_split); + if (S.is_error()) { + abort(std::move(S)); + return; + } + CHECK(prev.size() == 1 || prev.size() == 2); + bool found = prev_id == prev[0] || (prev.size() == 2 && prev_id == prev[1]); + if (!found) { + abort(td::Status::Error("invalid proof chain: prev block is not in prev blocks list")); + return; + } + } + links_.emplace_back(prev_id, std::move(proof)); + cur_id = prev_id; + } + } catch (vm::VmVirtError& err) { + abort(err.as_status()); + return; + } + if (cur_id != id_) { + abort(td::Status::Error("got invalid proof chain")); + return; + } + + if (mc_id_.seqno() > from_.seqno()) { + abort(td::Status::Error("from mc block is too old")); + return; + } + + client_.send_query( + ton::lite_api::liteServer_getBlockProof(0x1001, ton::create_tl_lite_block_id(from_), + ton::create_tl_lite_block_id(mc_id_)), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetShardBlockProof::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetShardBlockProof::got_mc_proof, R.move_as_ok()); + } + }); + } + + void got_mc_proof(lite_api_ptr result) { + auto R = liteclient::deserialize_proof_chain(std::move(result)); + if (R.is_error()) { + abort(R.move_as_error()); + return; + } + auto chain = R.move_as_ok(); + if (chain->from != from_ || chain->to != mc_id_ || !chain->complete || chain->link_count() > 1) { + abort(td::Status::Error("got invalid proof chain")); + return; + } + auto S = chain->validate(); + if (S.is_error()) { + abort(std::move(S)); + return; + } + + std::vector> links; + std::vector> mc_proof; + for (const auto& p : links_) { + links.push_back( + ton::create_tl_object(to_tonlib_api(p.first), p.second.as_slice().str())); + } + if (chain->link_count() == 1) { + auto& link = chain->last_link(); + td::BufferSlice dest_proof = vm::std_boc_serialize(link.dest_proof).move_as_ok(); + td::BufferSlice proof = vm::std_boc_serialize(link.proof).move_as_ok(); + td::BufferSlice state_proof = vm::std_boc_serialize(link.state_proof).move_as_ok(); + mc_proof.push_back(ton::create_tl_object( + link.is_key, to_tonlib_api(link.from), to_tonlib_api(link.to), dest_proof.as_slice().str(), + proof.as_slice().str(), state_proof.as_slice().str())); + } + + promise_.set_result(ton::create_tl_object( + to_tonlib_api(from_), to_tonlib_api(mc_id_), std::move(links), std::move(mc_proof))); + stop(); + } + + void abort(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + ton::BlockIdExt id_, from_, mc_id_; + td::actor::ActorShared<> parent_; + td::Promise> promise_; + ExtClient client_; + std::vector> links_; +}; + TonlibClient::TonlibClient(td::unique_ptr callback) : callback_(std::move(callback)) { } TonlibClient::~TonlibClient() = default; @@ -2170,10 +2487,14 @@ td::Result to_std_address(td::Ref cs) { return TRY_VM(to_std_address_or_throw(std::move(cs))); } struct ToRawTransactions { - explicit ToRawTransactions(td::optional private_key) : private_key_(std::move(private_key)) { + explicit ToRawTransactions(td::optional private_key, bool try_decode_messages = true) + : private_key_(std::move(private_key)) + , try_decode_messages_(try_decode_messages) { } td::optional private_key_; + bool try_decode_messages_; + td::Result> to_raw_message_or_throw(td::Ref cell) { block::gen::Message::Record message; if (!tlb::type_unpack_cell(cell, block::gen::t_Message_Any, message)) { @@ -2192,7 +2513,7 @@ struct ToRawTransactions { auto get_data = [body = std::move(body), body_cell, this](td::Slice salt) mutable { tonlib_api::object_ptr data; - if (body->size() >= 32 && static_cast(body->prefetch_long(32)) <= 1) { + if (try_decode_messages_ && body->size() >= 32 && static_cast(body->prefetch_long(32)) <= 1) { auto type = body.write().fetch_long(32); td::Status status; @@ -2203,7 +2524,6 @@ struct ToRawTransactions { if (type == 0) { data = tonlib_api::make_object(r_body_message.move_as_ok()); } else { - LOG(ERROR) << "TRY DECRYPT"; auto encrypted_message = r_body_message.move_as_ok(); auto r_decrypted_message = [&]() -> td::Result { if (!private_key_) { @@ -2464,13 +2784,56 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactions& request, auto actor_id = actor_id_++; actors_[actor_id] = td::actor::create_actor( - "GetTransactionHistory", client_.get_client(), account_address, lt, hash, actor_shared(this, actor_id), + "GetTransactionHistory", client_.get_client(), account_address, lt, hash, 10, actor_shared(this, actor_id), promise.wrap([private_key = std::move(private_key)](auto&& x) mutable { return ToRawTransactions(std::move(private_key)).to_raw_transactions(std::move(x)); })); return td::Status::OK(); } +td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request, + td::Promise>&& promise) { + if (!request.account_address_) { + return TonlibError::EmptyField("account_address"); + } + if (!request.from_transaction_id_) { + return TonlibError::EmptyField("from_transaction_id"); + } + TRY_RESULT(account_address, get_account_address(request.account_address_->account_address_)); + td::optional private_key; + if (request.private_key_) { + TRY_RESULT(input_key, from_tonlib(*request.private_key_)); + //NB: optional has lot of problems. We use emplace to migitate them + td::optional o_status; + //NB: rely on (and assert) that GetPrivateKey is a synchonous request + make_request(int_api::GetPrivateKey{std::move(input_key)}, [&](auto&& r_key) { + if (r_key.is_error()) { + o_status.emplace(r_key.move_as_error()); + return; + } + o_status.emplace(td::Status::OK()); + private_key = td::Ed25519::PrivateKey(std::move(r_key.move_as_ok().private_key)); + }); + TRY_STATUS(o_status.unwrap()); + } + auto lt = request.from_transaction_id_->lt_; + auto hash_str = request.from_transaction_id_->hash_; + if (hash_str.size() != 32) { + return td::Status::Error(400, "Invalid transaction id hash size"); + } + td::Bits256 hash; + hash.as_slice().copy_from(hash_str); + td::int32 count = request.count_ ? request.count_ : 10; + + auto actor_id = actor_id_++; + actors_[actor_id] = td::actor::create_actor( + "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), + promise.wrap([private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable { + return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x)); + })); + return td::Status::OK(); +} + td::Status TonlibClient::do_request(const tonlib_api::getAccountState& request, td::Promise>&& promise) { if (!request.account_address_) { @@ -4223,6 +4586,12 @@ auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result to_block_id(const tonlib_api::ton_blockIdExt& blk) { + TRY_RESULT(root_hash, to_bits256(blk.root_hash_, "blk.root_hash")) + TRY_RESULT(file_hash, to_bits256(blk.file_hash_, "blk.file_hash")) + return ton::BlockIdExt(blk.workchain_, blk.shard_, blk.seqno_, root_hash, file_hash); +} + td::Status TonlibClient::do_request(const tonlib_api::getConfigParam& request, td::Promise>&& promise) { TRY_RESULT(lite_block, to_lite_api(*request.id_)) @@ -4261,35 +4630,36 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainInfo& } td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request, - td::Promise>&& promise) { + td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) client_.send_query(ton::lite_api::liteServer_getAllShardsInfo(std::move(block)), - promise.wrap([](lite_api_ptr&& all_shards_info) { - td::BufferSlice proof = std::move((*all_shards_info).proof_); - td::BufferSlice data = std::move((*all_shards_info).data_); - if (data.empty()) { - //return td::Status::Error("shard configuration is empty"); - } else { - auto R = vm::std_boc_deserialize(data.clone()); - if (R.is_error()) { - //return td::Status::Error("cannot deserialize shard configuration"); - } - auto root = R.move_as_ok(); - block::ShardConfig sh_conf; - if (!sh_conf.unpack(vm::load_cell_slice_ref(root))) { - //return td::Status::Error("cannot extract shard block list from shard configuration"); - } else { - auto ids = sh_conf.get_shard_hash_ids(true); - tonlib_api::blocks_shards shards; - for (auto id : ids) { - auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id)); - if (ref.not_null()) { - shards.shards_.push_back(to_tonlib_api(ref->top_block_id())); - } - } + promise.wrap([](lite_api_ptr&& all_shards_info) + -> td::Result> { + td::BufferSlice proof = std::move((*all_shards_info).proof_); + td::BufferSlice data = std::move((*all_shards_info).data_); + if (data.empty()) { + return td::Status::Error("shard configuration is empty"); + } else { + auto R = vm::std_boc_deserialize(data.clone()); + if (R.is_error()) { + return R.move_as_error_prefix("cannot deserialize shard configuration: "); + } + auto root = R.move_as_ok(); + block::ShardConfig sh_conf; + if (!sh_conf.unpack(vm::load_cell_slice_ref(root))) { + return td::Status::Error("cannot extract shard block list from shard configuration"); + } else { + auto ids = sh_conf.get_shard_hash_ids(true); + tonlib_api::blocks_shards shards; + for (auto id : ids) { + auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id)); + if (ref.not_null()) { + shards.shards_.push_back(to_tonlib_api(ref->top_block_id())); + } + } return tonlib_api::make_object(std::move(shards)); - } - } + } + } })); return td::Status::OK(); } @@ -4424,6 +4794,28 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainBlockSignatures& request, + td::Promise>&& promise) { + auto actor_id = actor_id_++; + actors_[actor_id] = td::actor::create_actor( + "GetMasterchainBlockSignatures", client_.get_client(), request.seqno_, actor_shared(this, actor_id), + std::move(promise)); + return td::Status::OK(); +} + +td::Status TonlibClient::do_request(const tonlib_api::blocks_getShardBlockProof& request, + td::Promise>&& promise) { + TRY_RESULT(id, to_block_id(*request.id_)); + ton::BlockIdExt from; + if (request.mode_ & 1) { + TRY_RESULT_ASSIGN(from, to_block_id(*request.id_)); + } + auto actor_id = actor_id_++; + actors_[actor_id] = td::actor::create_actor("GetShardBlockProof", client_.get_client(), id, from, + actor_shared(this, actor_id), std::move(promise)); + return td::Status::OK(); +} + void TonlibClient::load_libs_from_disk() { LOG(DEBUG) << "loading libraries from disk cache"; auto r_data = kv_->get("tonlib.libcache"); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 10c83f21..800d8c80 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -236,6 +236,8 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); td::Status do_request(tonlib_api::raw_getTransactions& request, td::Promise>&& promise); + td::Status do_request(tonlib_api::raw_getTransactionsV2& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::getAccountState& request, td::Promise>&& promise); @@ -362,6 +364,10 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); td::Status do_request(const tonlib_api::blocks_getBlockHeader& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::blocks_getMasterchainBlockSignatures& request, + td::Promise>&& promise); + td::Status do_request(const tonlib_api::blocks_getShardBlockProof& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::getConfigParam& request, td::Promise>&& promise); diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index be550454..e889234a 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -430,6 +430,7 @@ class TonlibCli : public td::actor::Actor { << "\t 'e' modifier - encrypt all messages\n" << "\t 'k' modifier - use fake key\n" << "\t 'c' modifier - just esmitate fees\n"; + td::TerminalIO::out() << "getmasterchainsignatures - get sigratures of masterchain block \n"; } else if (cmd == "genkey") { generate_key(); } else if (cmd == "exit" || cmd == "quit") { @@ -510,6 +511,9 @@ class TonlibCli : public td::actor::Actor { auto key = parser.read_word(); auto init_key = parser.read_word(); guess_account(key, init_key, std::move(cmd_promise)); + } else if (cmd == "getmasterchainsignatures") { + auto seqno = parser.read_word(); + run_get_masterchain_block_signatures(seqno, std::move(cmd_promise)); } else { cmd_promise.set_error(td::Status::Error(PSLICE() << "Unkwnown query `" << cmd << "`")); } @@ -2096,6 +2100,18 @@ class TonlibCli : public td::actor::Actor { })); } + void run_get_masterchain_block_signatures(td::Slice seqno_s, td::Promise promise) { + TRY_RESULT_PROMISE(promise, seqno, td::to_integer_safe(seqno_s)); + send_query(make_object(seqno), promise.wrap([](auto signatures) { + td::TerminalIO::out() << "Signatures: " << signatures->signatures_.size() << "\n"; + for (const auto& s : signatures->signatures_) { + td::TerminalIO::out() << " " << s->node_id_short_ << " : " << td::base64_encode(td::Slice(s->signature_)) + << "\n"; + } + return td::Unit(); + })); + } + void get_history2(td::Slice key, td::Result> r_state, td::Promise promise) { TRY_RESULT_PROMISE(promise, state, std::move(r_state)); diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 8e2e208b..cab34777 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1009,6 +1009,56 @@ td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { return td::Status::OK(); } +td::Status GetPerfTimerStatsJsonQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetPerfTimerStatsJsonQuery::send() { + auto b = ton::create_serialize_tl_object(""); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetPerfTimerStatsJsonQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + std::ofstream sb(file_name_); + + sb << "{"; + bool gtail = false; + for (const auto &v : f->stats_) { + if (gtail) { + sb << ","; + } else { + gtail = true; + } + + sb << "\n \"" << v->name_ << "\": {"; + bool tail = false; + for (const auto &stat : v->stats_) { + if (tail) { + sb << ","; + } else { + tail = true; + } + + sb << "\n \"" << stat->time_ << "\": ["; + sb << "\n " << stat->min_ << ","; + sb << "\n " << stat->avg_ << ","; + sb << "\n " << stat->max_; + sb << "\n ]"; + } + sb << "\n }"; + } + sb << "\n}\n"; + sb << std::flush; + + td::TerminalIO::output(std::string("wrote stats to " + file_name_ + "\n")); + return td::Status::OK(); +} + td::Status GetValidatorSessionsInfoQuery::run() { TRY_STATUS(tokenizer_.check_endl()); return td::Status::OK(); diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index cfb8850d..316cc5e7 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1075,6 +1075,28 @@ class ImportShardOverlayCertificateQuery : public Query { std::string in_file_; }; +class GetPerfTimerStatsJsonQuery : public Query { + public: + GetPerfTimerStatsJsonQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getperftimerstatsjson"; + } + static std::string get_help() { + return "getperftimerstatsjson \tgets min, average and max event processing time for last 60, 300 and 3600 seconds and writes to json file"; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + class GetValidatorSessionsInfoQuery : public Query { public: GetValidatorSessionsInfoQuery(td::actor::ActorId console, Tokenizer tokenizer) diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 0ce81fb4..a1a0135b 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -140,6 +140,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index bb3bd059..14a6961f 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -67,9 +67,11 @@ #include #include #include +#include #include #include "git.h" + Config::Config() { out_port = 3278; full_node = ton::PublicKeyHash::zero(); @@ -3355,6 +3357,57 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverla }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), query = std::move(query)](td::Result> R) mutable { + const std::vector times{60, 300, 3600}; + double now = td::Time::now(); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + auto r = R.move_as_ok(); + std::vector> by_name; + for (const auto &stats : r) { + if (stats.name == query.name_ || query.name_.empty()) { + std::vector> by_time; + for (const auto &t : times) { + double min = std::numeric_limits::lowest(); + double max = std::numeric_limits::max(); + double sum = 0; + int cnt = 0; + for (const auto &stat : stats.stats) { + double time = stat.first; + double duration = stat.second; + if (now - time <= static_cast(t)) { + min = td::min(min, duration); + max = td::max(max, duration); + sum += duration; + ++cnt; + } + } + by_time.push_back(ton::create_tl_object(t, min, sum / static_cast(cnt), max)); + } + by_name.push_back(ton::create_tl_object(stats.name, std::move(by_time))); + } + } + promise.set_value(ton::create_serialize_tl_object(std::move(by_name))); + } + }); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, std::move(P)); +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 69ef5cb0..71276e5d 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -437,6 +437,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-session/validator-session-description.cpp b/validator-session/validator-session-description.cpp index 001f0405..43d4a851 100644 --- a/validator-session/validator-session-description.cpp +++ b/validator-session/validator-session-description.cpp @@ -167,20 +167,28 @@ void ValidatorSessionDescriptionImpl::update_hash(const RootObject *obj, HashTyp } void *ValidatorSessionDescriptionImpl::alloc(size_t size, size_t align, bool temp) { + CHECK(align && !(align & (align - 1))); // align should be a power of 2 + auto get_padding = [&](const uint8_t* ptr) { + return (-(size_t)ptr) & (align - 1); + }; if (temp) { + pdata_temp_ptr_ += get_padding(pdata_temp_ + pdata_temp_ptr_); auto s = pdata_temp_ptr_; pdata_temp_ptr_ += size; CHECK(s + size <= pdata_temp_size_); return static_cast(pdata_temp_ + s); } else { while (true) { - auto s = pdata_perm_ptr_; - pdata_perm_ptr_ += size; - - if (pdata_perm_ptr_ <= pdata_perm_.size() * pdata_perm_size_) { - return static_cast(pdata_perm_[s / pdata_perm_size_] + (s % pdata_perm_size_)); + size_t idx = pdata_perm_ptr_ / pdata_perm_size_; + if (idx < pdata_perm_.size()) { + auto ptr = pdata_perm_[idx] + (pdata_perm_ptr_ % pdata_perm_size_); + pdata_perm_ptr_ += get_padding(ptr); + ptr += get_padding(ptr); + pdata_perm_ptr_ += size; + if (pdata_perm_ptr_ <= pdata_perm_.size() * pdata_perm_size_) { + return static_cast(ptr); + } } - pdata_perm_.push_back(new td::uint8[pdata_perm_size_]); } } diff --git a/validator/apply-block.hpp b/validator/apply-block.hpp index 67ee3911..95deb025 100644 --- a/validator/apply-block.hpp +++ b/validator/apply-block.hpp @@ -46,7 +46,10 @@ class ApplyBlock : public td::actor::Actor { , masterchain_block_id_(masterchain_block_id) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("applyblock", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "applyblock", duration); + }) { } static constexpr td::uint32 apply_block_priority() { @@ -78,7 +81,7 @@ class ApplyBlock : public td::actor::Actor { BlockHandle handle_; td::Ref state_; - td::PerfWarningTimer perf_timer_{"applyblock", 0.1}; + td::PerfWarningTimer perf_timer_; }; } // namespace validator diff --git a/validator/downloaders/wait-block-data.hpp b/validator/downloaders/wait-block-data.hpp index 73097d30..9a03b1cb 100644 --- a/validator/downloaders/wait-block-data.hpp +++ b/validator/downloaders/wait-block-data.hpp @@ -35,7 +35,10 @@ class WaitBlockData : public td::actor::Actor { , priority_(priority) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("waitdata", 1.0, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration); + }) { } void update_timeout(td::Timestamp timeout, td::uint32 priority) { @@ -74,7 +77,7 @@ class WaitBlockData : public td::actor::Actor { bool is_hardfork_ = false; td::Timestamp try_read_static_file_ = td::Timestamp::now(); - //td::PerfWarningTimer perf_timer_{"waitdata", 1.0}; + td::PerfWarningTimer perf_timer_; }; } // namespace validator diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index fc25eeec..58c3740f 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -34,7 +34,10 @@ class WaitBlockState : public td::actor::Actor { , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) - , persistent_state_desc_(std::move(persistent_state_desc)) { + , persistent_state_desc_(std::move(persistent_state_desc)) + , perf_timer_("waitstate", 1.0, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); + }) { } void abort_query(td::Status reason); @@ -83,7 +86,7 @@ class WaitBlockState : public td::actor::Actor { bool reading_from_db_ = false; td::Timestamp next_static_file_attempt_; - //td::PerfWarningTimer perf_timer_{"waitstate", 1.0}; + td::PerfWarningTimer perf_timer_{"waitstate", 1.0}; bool check_persistent_state_desc() const { if (persistent_state_desc_.is_null()) { diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index c3e20c0e..6b63807b 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -54,7 +54,10 @@ AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std:: , send_broadcast_(send_broadcast) , apply_(apply) , manager_(manager) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("acceptblock", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "acceptblock", duration); + }) { state_keep_old_hash_.clear(); state_old_hash_.clear(); state_hash_.clear(); @@ -72,7 +75,10 @@ AcceptBlockQuery::AcceptBlockQuery(AcceptBlockQuery::IsFake fake, BlockIdExt id, , is_fork_(false) , send_broadcast_(false) , manager_(manager) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("acceptblock", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "acceptblock", duration); + }) { state_keep_old_hash_.clear(); state_old_hash_.clear(); state_hash_.clear(); @@ -87,7 +93,10 @@ AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref top_block_descr_; - td::PerfWarningTimer perf_timer_{"acceptblock", 0.1}; + td::PerfWarningTimer perf_timer_; bool fatal_error(std::string msg, int code = -666); static bool check_send_error(td::actor::ActorId SelfId, td::Status error); diff --git a/validator/impl/check-proof.hpp b/validator/impl/check-proof.hpp index 4461b5e1..5c9a29b5 100644 --- a/validator/impl/check-proof.hpp +++ b/validator/impl/check-proof.hpp @@ -47,7 +47,10 @@ class CheckProof : public td::actor::Actor { , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) - , skip_check_signatures_(skip_check_signatures) { + , skip_check_signatures_(skip_check_signatures) + , perf_timer_("checkproof", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); + }) { } CheckProof(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool skip_check_signatures, td::Ref known_state) @@ -58,7 +61,10 @@ class CheckProof : public td::actor::Actor { , timeout_(timeout) , promise_(std::move(promise)) , state_(std::move(known_state)) - , skip_check_signatures_(skip_check_signatures) { + , skip_check_signatures_(skip_check_signatures) + , perf_timer_("checkproof", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); + }) { } CheckProof(BlockIdExt id, td::Ref proof_link, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) @@ -67,7 +73,10 @@ class CheckProof : public td::actor::Actor { , proof_(std::move(proof_link)) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("checkproof", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); + }) { } private: @@ -114,7 +123,7 @@ class CheckProof : public td::actor::Actor { bool skip_check_signatures_{false}; bool sig_ok_{false}; - td::PerfWarningTimer perf_timer_{"checkproof", 0.1}; + td::PerfWarningTimer perf_timer_; static bool check_send_error(td::actor::ActorId SelfId, td::Status error); template diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 58f87fce..daa1a21b 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -213,7 +213,7 @@ class Collator final : public td::actor::Actor { std::unique_ptr block_candidate; - td::PerfWarningTimer perf_timer_{"collate", 0.1}; + td::PerfWarningTimer perf_timer_; // block::Account* lookup_account(td::ConstBitPtr addr) const; std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account, diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 5d0d2972..966e638a 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -66,7 +66,10 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , validator_set_(std::move(validator_set)) , manager(manager) , timeout(timeout) - , main_promise(std::move(promise)) { + , main_promise(std::move(promise)) + , perf_timer_("collate", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); + }) { } void Collator::start_up() { diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 023a84ae..fadc596b 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -201,6 +201,9 @@ void LiteQuery::start_up() { [&](lite_api::liteServer_getLibraries& q) { this->perform_getLibraries(q.library_list_); }, + [&](lite_api::liteServer_getShardBlockProof& q) { + this->perform_getShardBlockProof(create_block_id(q.id_)); + }, [&](auto& obj) { this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); })); } @@ -269,15 +272,21 @@ void LiteQuery::perform_getBlock(BlockIdExt blkid) { fatal_error("invalid BlockIdExt"); return; } - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getBlock, blkid, - res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [manager = manager_, Self = actor_id(this), blkid](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getBlock, blkid, + res.move_as_ok()); + } + }); + }); } void LiteQuery::continue_getBlock(BlockIdExt blkid, Ref block) { @@ -295,15 +304,21 @@ void LiteQuery::perform_getBlockHeader(BlockIdExt blkid, int mode) { fatal_error("invalid BlockIdExt"); return; } - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self = actor_id(this), blkid, mode](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getBlockHeader, blkid, - mode, res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getBlockHeader, blkid, + mode, res.move_as_ok()); + } + }); + }); } static bool visit(Ref cell); @@ -409,27 +424,33 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { fatal_error("cannot request total state: possibly too large"); return; } - if (blkid.id.seqno) { - td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getState, blkid, - res.move_as_ok()); - } - }); - } else { - td::actor::send_closure_later(manager_, &ValidatorManager::get_zero_state, blkid, - [Self = actor_id(this), blkid](td::Result res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getZeroState, blkid, - res.move_as_ok()); - } - }); - } + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + if (blkid.id.seqno) { + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getState, blkid, + res.move_as_ok()); + } + }); + } else { + td::actor::send_closure_later(manager, &ValidatorManager::get_zero_state, blkid, + [=](td::Result res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getZeroState, blkid, + res.move_as_ok()); + } + }); + } + }); } void LiteQuery::continue_getState(BlockIdExt blkid, Ref state) { @@ -481,6 +502,23 @@ void LiteQuery::perform_sendMessage(td::BufferSlice data) { }); } +void LiteQuery::get_block_handle_checked(BlockIdExt blkid, td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto handle = R.move_as_ok(); + if (handle->is_applied()) { + promise.set_result(std::move(handle)); + } else { + promise.set_error(td::Status::Error("block is not applied")); + } + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, blkid, false, std::move(P)); +} + bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { if (!blkid.is_masterchain() || !blkid.is_valid_full()) { return fatal_error("reference block must belong to the masterchain"); @@ -579,16 +617,22 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) { } blk_id_ = blkid; ++pending_; - td::actor::send_closure_later( - manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); - } else { - td::actor::send_closure_later(Self, &LiteQuery::got_block_state, blkid, res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later( + manager, &ValidatorManager::get_shard_state_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); + } else { + td::actor::send_closure_later(Self, &LiteQuery::got_block_state, blkid, res.move_as_ok()); + } + }); + }); return true; } @@ -601,16 +645,22 @@ bool LiteQuery::request_block_data(BlockIdExt blkid) { } blk_id_ = blkid; ++pending_; - td::actor::send_closure_later( - manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); - } else { - td::actor::send_closure_later(Self, &LiteQuery::got_block_data, blkid, res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); + } else { + td::actor::send_closure_later(Self, &LiteQuery::got_block_data, blkid, res.move_as_ok()); + } + }); + }); return true; } @@ -646,16 +696,23 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) { }); }); } else { - td::actor::send_closure_later( - manager_, &ValidatorManager::get_block_proof_link_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : ")); - } else { - td::actor::send_closure_later(Self, &LiteQuery::got_block_proof_link, blkid, res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_proof_link_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure( + Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : ")); + } else { + td::actor::send_closure_later(Self, &LiteQuery::got_block_proof_link, blkid, res.move_as_ok()); + } + }); + }); } return true; } @@ -672,16 +729,22 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { } blk_id_ = blkid; ++pending_; - td::actor::send_closure_later( - manager_, &ValidatorManager::get_zero_state, blkid, - [Self = actor_id(this), blkid](td::Result res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : ")); - } else { - td::actor::send_closure_later(Self, &LiteQuery::got_zero_state, blkid, res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later( + manager, &ValidatorManager::get_zero_state, blkid, + [=](td::Result res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : ")); + } else { + td::actor::send_closure_later(Self, &LiteQuery::got_zero_state, blkid, res.move_as_ok()); + } + }); + }); return true; } @@ -1397,6 +1460,11 @@ void LiteQuery::continue_getTransactions(unsigned remaining, bool exact) { td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, res.move_as_error(), ton::BlockIdExt{}); } else { auto handle = res.move_as_ok(); + if (!handle->is_applied()) { + td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, td::Status::Error("block is not applied"), + ton::BlockIdExt{}); + return; + } LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, [Self, blkid = handle->id(), remaining](td::Result> res) { @@ -1722,6 +1790,10 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto handle = res.move_as_ok(); + if (!handle->is_applied()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block is not applied")); + return; + } LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, [Self, blkid = handle->id(), mode](td::Result> res) { @@ -2363,5 +2435,135 @@ void LiteQuery::continue_getValidatorStats(int mode, int limit, Bits256 start_af finish_query(std::move(b)); } +void LiteQuery::perform_getShardBlockProof(BlockIdExt blkid) { + LOG(INFO) << "started a getMasterchainInfo(" << blkid.to_str() << ") liteserver query"; + if (!blkid.is_valid_ext()) { + fatal_error("invalid block id"); + return; + } + if (blkid.is_masterchain()) { + LOG(INFO) << "getShardBlockProof() query completed"; + auto b = create_serialize_tl_object( + create_tl_lite_block_id(blkid), std::vector>()); + finish_query(std::move(b)); + return; + } + blk_id_ = blkid; + get_block_handle_checked(blkid, [manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + ConstBlockHandle handle = R.move_as_ok(); + if (!handle->inited_masterchain_ref_block()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block doesn't have masterchain ref")); + return; + } + AccountIdPrefixFull pfx{masterchainId, shardIdAll}; + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_by_seqno_from_db, pfx, handle->masterchain_ref_block(), + [Self, manager](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + } else { + ConstBlockHandle handle = R.move_as_ok(); + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_data_from_db, handle, [Self](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getShardBlockProof, R.move_as_ok(), + std::vector>()); + } + }); + } + }); + }); +} + +void LiteQuery::continue_getShardBlockProof(Ref cur_block, + std::vector> result) { + BlockIdExt cur_id = cur_block->block_id(); + BlockIdExt prev_id; + vm::MerkleProofBuilder mpb{cur_block->root_cell()}; + if (cur_id.is_masterchain()) { + base_blk_id_ = cur_id; + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + if (!tlb::unpack_cell(mpb.root(), blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() || + !tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) { + fatal_error("cannot unpack header of block "s + cur_id.to_str()); + return; + } + block::ShardConfig shards(mc_extra.shard_hashes->prefetch_ref()); + ShardIdFull shard_id = blk_id_.shard_full(); + shard_id.shard = (shard_id.shard & ~(1 << (63 - shard_id.pfx_len()))) | 1; + Ref shard_hash = shards.get_shard_hash(shard_id, false); + if (shard_hash.is_null()) { + fatal_error("shard not found"); + return; + } + prev_id = shard_hash->top_block_id(); + } else { + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + td::Status S = block::unpack_block_prev_blk_try(mpb.root(), cur_id, prev, mc_blkid, after_split); + if (S.is_error()) { + fatal_error(std::move(S)); + return; + } + bool found = false; + for (const BlockIdExt& id : prev) { + if (shard_intersects(id.shard_full(), blk_id_.shard_full())) { + found = true; + prev_id = id; + break; + } + } + if (!found) { + fatal_error("failed to find block chain"); + return; + } + } + auto proof = mpb.extract_proof_boc(); + if (proof.is_error()) { + fatal_error(proof.move_as_error_prefix("cannot serialize Merkle proof : ")); + return; + } + result.emplace_back(prev_id, proof.move_as_ok()); + + if (prev_id == blk_id_) { + CHECK(base_blk_id_.is_masterchain()); + std::vector> links; + for (auto& p : result) { + links.push_back( + create_tl_object(create_tl_lite_block_id(p.first), std::move(p.second))); + } + LOG(INFO) << "getShardBlockProof() query completed"; + auto b = create_serialize_tl_object(create_tl_lite_block_id(base_blk_id_), + std::move(links)); + finish_query(std::move(b)); + return; + } + if (result.size() == 8) { + // Chains of shardblocks between masterchain blocks can't be longer than 8 (see collator.cpp:991) + fatal_error("proof chain is too long"); + return; + } + + td::actor::send_closure_later( + manager_, &ValidatorManager::get_block_data_from_db_short, prev_id, + [Self = actor_id(this), result = std::move(result)](td::Result> R) mutable { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getShardBlockProof, R.move_as_ok(), + std::move(result)); + } + }); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index f1918ceb..47970aae 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -145,6 +145,9 @@ class LiteQuery : public td::actor::Actor { bool construct_proof_link_back_cont(ton::BlockIdExt cur, ton::BlockIdExt next); bool adjust_last_proof_link(ton::BlockIdExt cur, Ref block_root); bool finish_proof_chain(ton::BlockIdExt id); + void perform_getShardBlockProof(BlockIdExt blkid); + void continue_getShardBlockProof(Ref cur_block, + std::vector> result); void load_prevKeyBlock(ton::BlockIdExt blkid, td::Promise>>); void continue_loadPrevKeyBlock(ton::BlockIdExt blkid, td::Result, BlockIdExt>> res, @@ -152,6 +155,7 @@ class LiteQuery : public td::actor::Actor { void finish_loadPrevKeyBlock(ton::BlockIdExt blkid, td::Result> res, td::Promise>> promise); + void get_block_handle_checked(BlockIdExt blkid, td::Promise promise); bool request_block_data(BlockIdExt blkid); bool request_block_state(BlockIdExt blkid); bool request_block_data_state(BlockIdExt blkid); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index f8a2708d..15249fe5 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -59,13 +59,16 @@ ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block , prev_blocks(std::move(prev)) , block_candidate(std::move(candidate)) , validator_set_(std::move(validator_set)) - , manager(std::move(manager)) + , manager(manager) , timeout(timeout) , main_promise(std::move(promise)) , is_fake_(mode & ValidateMode::fake) , full_collated_data_(mode & ValidateMode::full_collated_data) , shard_pfx_(shard_.shard) - , shard_pfx_len_(ton::shard_prefix_length(shard_)) { + , shard_pfx_len_(ton::shard_prefix_length(shard_)) + , perf_timer_("validateblock", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validateblock", duration); + }) { } void ValidateQuery::alarm() { @@ -2392,10 +2395,6 @@ bool ValidateQuery::precheck_one_account_update(td::ConstBitPtr acc_id, Ref> lib_publishers_, lib_publishers2_; - td::PerfWarningTimer perf_timer_{"validateblock", 0.1}; + td::PerfWarningTimer perf_timer_; static constexpr td::uint32 priority() { return 2; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 1ab6fb09..c4403e94 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -366,6 +366,13 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void prepare_perf_timer_stats(td::Promise> promise) override { + UNREACHABLE(); + } + + void add_perf_timer_stat(std::string name, double duration) override { + } + void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) override { UNREACHABLE(); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 3d20e494..bd30b1de 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -426,6 +426,13 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void prepare_perf_timer_stats(td::Promise> promise) override { + UNREACHABLE(); + } + + void add_perf_timer_stat(std::string name, double duration) override { + } + void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 2a17af5e..183dc8c1 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2613,6 +2613,24 @@ void ValidatorManagerImpl::prepare_stats(td::Promise> promise) { + promise.set_value(std::vector(perf_timer_stats)); +} + +void ValidatorManagerImpl::add_perf_timer_stat(std::string name, double duration) { + for (auto &s : perf_timer_stats) { + if (s.name == name) { + double now = td::Time::now(); + while (!s.stats.empty() && s.stats.front().first < now - 3600.0) { + s.stats.pop_front(); + } + s.stats.push_back({td::Time::now(), duration}); + return; + } + } + perf_timer_stats.push_back({name, {{td::Time::now(), duration}}}); +} + void ValidatorManagerImpl::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) { td::actor::send_closure(db_, &Db::truncate, seqno, std::move(handle), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 3b32cfcd..136e5e8a 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -265,6 +265,8 @@ class ValidatorManagerImpl : public ValidatorManager { std::map, std::vector>>> pending_masterchain_states_; + std::vector perf_timer_stats; + void new_masterchain_block(); void update_shards(); void update_shard_blocks(); @@ -542,6 +544,9 @@ class ValidatorManagerImpl : public ValidatorManager { void prepare_stats(td::Promise>> promise) override; + void prepare_perf_timer_stats(td::Promise> promise) override; + void add_perf_timer_stat(std::string name, double duration) override; + void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) override; void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; diff --git a/validator/validate-broadcast.hpp b/validator/validate-broadcast.hpp index 8a6fb149..74496fa3 100644 --- a/validator/validate-broadcast.hpp +++ b/validator/validate-broadcast.hpp @@ -44,7 +44,7 @@ class ValidateBroadcast : public td::actor::Actor { td::Ref proof_link_; BlockHandle handle_; - td::PerfWarningTimer perf_timer_{"validatebroadcast", 0.1}; + td::PerfWarningTimer perf_timer_; bool exact_key_block_handle_; td::Ref key_proof_link_; @@ -60,7 +60,10 @@ class ValidateBroadcast : public td::actor::Actor { , last_known_masterchain_block_handle_(std::move(last_known_masterchain_block_handle)) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("validatebroadcast", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validatebroadcast", duration); + }) { } void start_up() override; diff --git a/validator/validator.h b/validator/validator.h index c52cbad7..c387caff 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include "td/actor/actor.h" @@ -45,6 +46,11 @@ class DownloadToken { virtual ~DownloadToken() = default; }; +struct PerfTimerStats { + std::string name; + std::deque> stats; // +}; + struct ValidatorManagerOptions : public td::CntObject { public: virtual BlockIdExt zero_block_id() const = 0; @@ -224,6 +230,10 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void run_ext_query(td::BufferSlice data, td::Promise promise) = 0; virtual void prepare_stats(td::Promise>> promise) = 0; + + virtual void prepare_perf_timer_stats(td::Promise> promise) = 0; + virtual void add_perf_timer_stat(std::string name, double duration) = 0; + virtual void get_validator_sessions_info( td::Promise> promise) = 0;